From 3749a2612a27e10ec5482e1e829b2944d5dce02b Mon Sep 17 00:00:00 2001 From: delorean Date: Wed, 21 Feb 2024 18:23:18 -0600 Subject: [PATCH] initial --- .clang-format | 11 + .editorconfig | 18 + .gitignore | 38 + 10gigE.md | 57 + CMakeLists.txt | 168 +++ CONTRIBUTING.md | 36 + checkFormat.sh | 20 + conf/blocklist.conf | 25 + conf/zmap.conf | 22 + examples/forge-socket/README | 4 + .../probe-modules/module_tcp_cisco_backdoor.c | 203 +++ examples/udp-probes/README | 51 + examples/udp-probes/ard_3283.pkt | Bin 0 -> 5 bytes examples/udp-probes/bacnet_47808.pkt | Bin 0 -> 18 bytes examples/udp-probes/bacnet_rpm_47808.pkt | Bin 0 -> 37 bytes examples/udp-probes/chargen_19.pkt | 1 + examples/udp-probes/citrix_1604.pkt | Bin 0 -> 30 bytes examples/udp-probes/coap_5683.pkt | 1 + examples/udp-probes/db2_523.pkt | Bin 0 -> 20 bytes examples/udp-probes/db2disco_523.pkt | Bin 0 -> 20 bytes examples/udp-probes/digi1_2362.pkt | Bin 0 -> 14 bytes examples/udp-probes/digi2_2362.pkt | Bin 0 -> 14 bytes examples/udp-probes/digi3_2362.pkt | Bin 0 -> 14 bytes examples/udp-probes/dns_53.pkt | Bin 0 -> 30 bytes .../dns_53_queryAwww.google.com.pkt | Bin 0 -> 32 bytes .../udp-probes/dns_53_queryAwww.google.it.pkt | Bin 0 -> 31 bytes examples/udp-probes/ipmi_623.pkt | Bin 0 -> 23 bytes examples/udp-probes/ldap_389.pkt | Bin 0 -> 53 bytes examples/udp-probes/mdns_5353.pkt | Bin 0 -> 46 bytes examples/udp-probes/memcache_11211.pkt | Bin 0 -> 21 bytes examples/udp-probes/mssql_1434.pkt | 1 + examples/udp-probes/nat_port_mapping_5351.pkt | Bin 0 -> 18 bytes examples/udp-probes/natpmp_5351.pkt | Bin 0 -> 4 bytes examples/udp-probes/netbios_137.pkt | Bin 0 -> 50 bytes examples/udp-probes/netis_53413.pkt | Bin 0 -> 18 bytes examples/udp-probes/ntp_123.pkt | Bin 0 -> 48 bytes examples/udp-probes/ntp_123_monlist.pkt | Bin 0 -> 192 bytes examples/udp-probes/openvpn_1194.pkt | Bin 0 -> 18 bytes examples/udp-probes/pca_nq_5632.pkt | 1 + examples/udp-probes/pca_st_5632.pkt | 1 + examples/udp-probes/pcanywhere_5632.pkt | Bin 0 -> 18 bytes examples/udp-probes/portmap_111.pkt | Bin 0 -> 40 bytes examples/udp-probes/qotd_17.pkt | 1 + examples/udp-probes/rdp_3389.pkt | Bin 0 -> 18 bytes examples/udp-probes/ripv1_520.pkt | Bin 0 -> 24 bytes examples/udp-probes/sentinel_5093.pkt | Bin 0 -> 6 bytes examples/udp-probes/sip_5060.pkt | 1 + examples/udp-probes/sip_options.tpl | 12 + examples/udp-probes/snmp1_161.pkt | Bin 0 -> 43 bytes examples/udp-probes/snmp2_161.pkt | Bin 0 -> 40 bytes examples/udp-probes/snmp3_161.pkt | Bin 0 -> 60 bytes examples/udp-probes/ssdp_1900.pkt | 5 + examples/udp-probes/tftp_69.pkt | Bin 0 -> 14 bytes examples/udp-probes/ubiquiti_10001.pkt | Bin 0 -> 18 bytes .../ubiquiti_discovery_v1_10001.pkt | Bin 0 -> 4 bytes .../ubiquiti_discovery_v2_10001.pkt | Bin 0 -> 4 bytes examples/udp-probes/upnp_1900.pkt | 7 + examples/udp-probes/valve_27015.pkt | Bin 0 -> 25 bytes examples/udp-probes/wdbrpc_17185.pkt | Bin 0 -> 64 bytes examples/udp-probes/wsd_3702.pkt | 3 + examples/udp-probes/wsd_malformed_3702.pkt | 1 + examples/udp-probes/xdmcp_177.pkt | Bin 0 -> 7 bytes format.sh | 24 + lib/CMakeLists.txt | 23 + lib/blocklist.c | 308 ++++ lib/blocklist.h | 50 + lib/cachehash.c | 275 ++++ lib/cachehash.h | 46 + lib/constraint.c | 416 ++++++ lib/constraint.h | 34 + lib/csv.c | 52 + lib/csv.h | 25 + lib/includes.h | 45 + lib/lockfd.c | 57 + lib/lockfd.h | 22 + lib/logger.c | 271 ++++ lib/logger.h | 67 + lib/pbm.c | 111 ++ lib/pbm.h | 31 + lib/queue.c | 109 ++ lib/queue.h | 49 + lib/random.c | 32 + lib/random.h | 25 + lib/rijndael-alg-fst.c | 1284 +++++++++++++++++ lib/rijndael-alg-fst.h | 53 + lib/types.h | 30 + lib/util.c | 371 +++++ lib/util.h | 74 + lib/xalloc.c | 55 + lib/xalloc.h | 30 + release-build.sh | 7 + scripts/check_manfile.py | 38 + src/.gitignore | 4 + src/CMakeLists.txt | 273 ++++ src/CMakeVersion.txt | 15 + src/aesrand.c | 71 + src/aesrand.h | 24 + src/constants.h | 16 + src/cyclic.c | 191 +++ src/cyclic.h | 45 + src/expression.c | 170 +++ src/expression.h | 54 + src/fieldset.c | 410 ++++++ src/fieldset.h | 147 ++ src/filter.c | 115 ++ src/filter.h | 23 + src/get_gateway-bsd.h | 224 +++ src/get_gateway-linux.h | 322 +++++ src/get_gateway.c | 30 + src/get_gateway.h | 20 + src/iterator.c | 144 ++ src/iterator.h | 34 + src/lexer.l | 36 + src/monitor.c | 491 +++++++ src/monitor.h | 18 + src/output_modules/module_csv.c | 126 ++ src/output_modules/module_csv.h | 14 + src/output_modules/module_json.c | 162 +++ src/output_modules/module_json.h | 19 + src/output_modules/output_modules.c | 41 + src/output_modules/output_modules.h | 45 + src/parser.y | 144 ++ src/ports.c | 63 + src/ports.h | 3 + src/probe_modules/module_bacnet.c | 195 +++ src/probe_modules/module_bacnet.h | 57 + src/probe_modules/module_dns.c | 1136 +++++++++++++++ src/probe_modules/module_dns.h | 69 + src/probe_modules/module_icmp_echo.c | 345 +++++ src/probe_modules/module_icmp_echo_time.c | 268 ++++ src/probe_modules/module_ntp.c | 252 ++++ src/probe_modules/module_ntp.h | 38 + src/probe_modules/module_tcp_synackscan.c | 225 +++ src/probe_modules/module_tcp_synscan.c | 234 +++ src/probe_modules/module_tcp_synscan.h | 38 + src/probe_modules/module_udp.c | 869 +++++++++++ src/probe_modules/module_udp.h | 96 ++ src/probe_modules/module_upnp.c | 272 ++++ src/probe_modules/packet.c | 252 ++++ src/probe_modules/packet.h | 243 ++++ src/probe_modules/probe_modules.c | 130 ++ src/probe_modules/probe_modules.h | 110 ++ src/recv-internal.h | 20 + src/recv-pcap.c | 164 +++ src/recv-pfring.c | 82 ++ src/recv.c | 238 +++ src/recv.h | 17 + src/send-bsd.h | 115 ++ src/send-linux.c | 121 ++ src/send-linux.h | 35 + src/send-pfring.h | 42 + src/send.c | 505 +++++++ src/send.h | 30 + src/shard.c | 193 +++ src/shard.h | 58 + src/socket-bsd.c | 88 ++ src/socket-linux.c | 36 + src/socket-pfring.c | 22 + src/socket.c | 31 + src/socket.h | 40 + src/state.c | 108 ++ src/state.h | 216 +++ src/summary.c | 363 +++++ src/summary.h | 24 + src/tests/test_harness.c | 97 ++ src/topt.ggo.in | 38 + src/topt_compat.c | 25 + src/utility.c | 71 + src/utility.h | 22 + src/validate.c | 54 + src/validate.h | 21 + src/zblocklist.1 | 60 + src/zblocklist.1.html | 120 ++ src/zblocklist.1.ronn | 53 + src/zblocklist.c | 263 ++++ src/zbopt.ggo.in | 43 + src/zbopt_compat.c | 17 + src/ziterate.1 | 74 + src/ziterate.1.html | 130 ++ src/ziterate.1.ronn | 66 + src/ziterate.c | 246 ++++ src/zitopt.ggo.in | 59 + src/zitopt_compat.c | 17 + src/zmap.1 | 294 ++++ src/zmap.1.html | 307 ++++ src/zmap.1.ronn | 309 ++++ src/zmap.c | 1060 ++++++++++++++ src/zmap_schema.py | 93 ++ src/zopt.ggo.in | 210 +++ src/zopt_compat.c | 17 + src/ztee.1 | 57 + src/ztee.1.html | 129 ++ src/ztee.1.ronn | 54 + src/ztee.c | 555 +++++++ src/ztopt.ggo.in | 18 + src/ztopt_compat.c | 17 + test/.gitignore | 6 + test/configs/blocklist_shard.conf | 25 + test/test-shard.sh | 57 + test/test_big_group.sh | 43 + test/test_sharding.py | 74 + test/test_zblocklist.py | 138 ++ 202 files changed, 20512 insertions(+) create mode 100644 .clang-format create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 10gigE.md create mode 100644 CMakeLists.txt create mode 100644 CONTRIBUTING.md create mode 100755 checkFormat.sh create mode 100644 conf/blocklist.conf create mode 100644 conf/zmap.conf create mode 100644 examples/forge-socket/README create mode 100644 examples/probe-modules/module_tcp_cisco_backdoor.c create mode 100644 examples/udp-probes/README create mode 100644 examples/udp-probes/ard_3283.pkt create mode 100644 examples/udp-probes/bacnet_47808.pkt create mode 100644 examples/udp-probes/bacnet_rpm_47808.pkt create mode 100644 examples/udp-probes/chargen_19.pkt create mode 100644 examples/udp-probes/citrix_1604.pkt create mode 100644 examples/udp-probes/coap_5683.pkt create mode 100644 examples/udp-probes/db2_523.pkt create mode 100644 examples/udp-probes/db2disco_523.pkt create mode 100644 examples/udp-probes/digi1_2362.pkt create mode 100644 examples/udp-probes/digi2_2362.pkt create mode 100644 examples/udp-probes/digi3_2362.pkt create mode 100644 examples/udp-probes/dns_53.pkt create mode 100644 examples/udp-probes/dns_53_queryAwww.google.com.pkt create mode 100644 examples/udp-probes/dns_53_queryAwww.google.it.pkt create mode 100644 examples/udp-probes/ipmi_623.pkt create mode 100644 examples/udp-probes/ldap_389.pkt create mode 100644 examples/udp-probes/mdns_5353.pkt create mode 100644 examples/udp-probes/memcache_11211.pkt create mode 100644 examples/udp-probes/mssql_1434.pkt create mode 100644 examples/udp-probes/nat_port_mapping_5351.pkt create mode 100644 examples/udp-probes/natpmp_5351.pkt create mode 100644 examples/udp-probes/netbios_137.pkt create mode 100644 examples/udp-probes/netis_53413.pkt create mode 100644 examples/udp-probes/ntp_123.pkt create mode 100644 examples/udp-probes/ntp_123_monlist.pkt create mode 100644 examples/udp-probes/openvpn_1194.pkt create mode 100644 examples/udp-probes/pca_nq_5632.pkt create mode 100644 examples/udp-probes/pca_st_5632.pkt create mode 100644 examples/udp-probes/pcanywhere_5632.pkt create mode 100644 examples/udp-probes/portmap_111.pkt create mode 100644 examples/udp-probes/qotd_17.pkt create mode 100644 examples/udp-probes/rdp_3389.pkt create mode 100644 examples/udp-probes/ripv1_520.pkt create mode 100644 examples/udp-probes/sentinel_5093.pkt create mode 100644 examples/udp-probes/sip_5060.pkt create mode 100644 examples/udp-probes/sip_options.tpl create mode 100644 examples/udp-probes/snmp1_161.pkt create mode 100644 examples/udp-probes/snmp2_161.pkt create mode 100644 examples/udp-probes/snmp3_161.pkt create mode 100644 examples/udp-probes/ssdp_1900.pkt create mode 100644 examples/udp-probes/tftp_69.pkt create mode 100644 examples/udp-probes/ubiquiti_10001.pkt create mode 100644 examples/udp-probes/ubiquiti_discovery_v1_10001.pkt create mode 100644 examples/udp-probes/ubiquiti_discovery_v2_10001.pkt create mode 100644 examples/udp-probes/upnp_1900.pkt create mode 100644 examples/udp-probes/valve_27015.pkt create mode 100644 examples/udp-probes/wdbrpc_17185.pkt create mode 100644 examples/udp-probes/wsd_3702.pkt create mode 100644 examples/udp-probes/wsd_malformed_3702.pkt create mode 100644 examples/udp-probes/xdmcp_177.pkt create mode 100755 format.sh create mode 100644 lib/CMakeLists.txt create mode 100644 lib/blocklist.c create mode 100644 lib/blocklist.h create mode 100644 lib/cachehash.c create mode 100644 lib/cachehash.h create mode 100644 lib/constraint.c create mode 100644 lib/constraint.h create mode 100644 lib/csv.c create mode 100644 lib/csv.h create mode 100644 lib/includes.h create mode 100644 lib/lockfd.c create mode 100644 lib/lockfd.h create mode 100644 lib/logger.c create mode 100644 lib/logger.h create mode 100644 lib/pbm.c create mode 100644 lib/pbm.h create mode 100644 lib/queue.c create mode 100644 lib/queue.h create mode 100644 lib/random.c create mode 100644 lib/random.h create mode 100644 lib/rijndael-alg-fst.c create mode 100644 lib/rijndael-alg-fst.h create mode 100644 lib/types.h create mode 100644 lib/util.c create mode 100644 lib/util.h create mode 100644 lib/xalloc.c create mode 100644 lib/xalloc.h create mode 100644 release-build.sh create mode 100755 scripts/check_manfile.py create mode 100644 src/.gitignore create mode 100644 src/CMakeLists.txt create mode 100644 src/CMakeVersion.txt create mode 100644 src/aesrand.c create mode 100644 src/aesrand.h create mode 100644 src/constants.h create mode 100644 src/cyclic.c create mode 100644 src/cyclic.h create mode 100644 src/expression.c create mode 100644 src/expression.h create mode 100644 src/fieldset.c create mode 100644 src/fieldset.h create mode 100644 src/filter.c create mode 100644 src/filter.h create mode 100644 src/get_gateway-bsd.h create mode 100644 src/get_gateway-linux.h create mode 100644 src/get_gateway.c create mode 100644 src/get_gateway.h create mode 100644 src/iterator.c create mode 100644 src/iterator.h create mode 100644 src/lexer.l create mode 100644 src/monitor.c create mode 100644 src/monitor.h create mode 100644 src/output_modules/module_csv.c create mode 100644 src/output_modules/module_csv.h create mode 100644 src/output_modules/module_json.c create mode 100644 src/output_modules/module_json.h create mode 100644 src/output_modules/output_modules.c create mode 100644 src/output_modules/output_modules.h create mode 100644 src/parser.y create mode 100644 src/ports.c create mode 100644 src/ports.h create mode 100644 src/probe_modules/module_bacnet.c create mode 100644 src/probe_modules/module_bacnet.h create mode 100644 src/probe_modules/module_dns.c create mode 100644 src/probe_modules/module_dns.h create mode 100644 src/probe_modules/module_icmp_echo.c create mode 100644 src/probe_modules/module_icmp_echo_time.c create mode 100644 src/probe_modules/module_ntp.c create mode 100644 src/probe_modules/module_ntp.h create mode 100644 src/probe_modules/module_tcp_synackscan.c create mode 100644 src/probe_modules/module_tcp_synscan.c create mode 100644 src/probe_modules/module_tcp_synscan.h create mode 100644 src/probe_modules/module_udp.c create mode 100644 src/probe_modules/module_udp.h create mode 100644 src/probe_modules/module_upnp.c create mode 100644 src/probe_modules/packet.c create mode 100644 src/probe_modules/packet.h create mode 100644 src/probe_modules/probe_modules.c create mode 100644 src/probe_modules/probe_modules.h create mode 100644 src/recv-internal.h create mode 100644 src/recv-pcap.c create mode 100644 src/recv-pfring.c create mode 100644 src/recv.c create mode 100644 src/recv.h create mode 100644 src/send-bsd.h create mode 100644 src/send-linux.c create mode 100644 src/send-linux.h create mode 100644 src/send-pfring.h create mode 100644 src/send.c create mode 100644 src/send.h create mode 100644 src/shard.c create mode 100644 src/shard.h create mode 100644 src/socket-bsd.c create mode 100644 src/socket-linux.c create mode 100644 src/socket-pfring.c create mode 100644 src/socket.c create mode 100644 src/socket.h create mode 100644 src/state.c create mode 100644 src/state.h create mode 100644 src/summary.c create mode 100644 src/summary.h create mode 100644 src/tests/test_harness.c create mode 100644 src/topt.ggo.in create mode 100644 src/topt_compat.c create mode 100644 src/utility.c create mode 100644 src/utility.h create mode 100644 src/validate.c create mode 100644 src/validate.h create mode 100644 src/zblocklist.1 create mode 100644 src/zblocklist.1.html create mode 100644 src/zblocklist.1.ronn create mode 100644 src/zblocklist.c create mode 100644 src/zbopt.ggo.in create mode 100644 src/zbopt_compat.c create mode 100644 src/ziterate.1 create mode 100644 src/ziterate.1.html create mode 100644 src/ziterate.1.ronn create mode 100644 src/ziterate.c create mode 100644 src/zitopt.ggo.in create mode 100644 src/zitopt_compat.c create mode 100644 src/zmap.1 create mode 100644 src/zmap.1.html create mode 100644 src/zmap.1.ronn create mode 100644 src/zmap.c create mode 100644 src/zmap_schema.py create mode 100644 src/zopt.ggo.in create mode 100644 src/zopt_compat.c create mode 100644 src/ztee.1 create mode 100644 src/ztee.1.html create mode 100644 src/ztee.1.ronn create mode 100644 src/ztee.c create mode 100644 src/ztopt.ggo.in create mode 100644 src/ztopt_compat.c create mode 100644 test/.gitignore create mode 100644 test/configs/blocklist_shard.conf create mode 100755 test/test-shard.sh create mode 100755 test/test_big_group.sh create mode 100644 test/test_sharding.py create mode 100644 test/test_zblocklist.py diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..2c50773 --- /dev/null +++ b/.clang-format @@ -0,0 +1,11 @@ +BasedOnStyle: LLVM +IndentWidth: 8 +UseTab: Always +BreakBeforeBraces: Linux +AllowShortIfStatementsOnASingleLine: false +IndentCaseLabels: false +DerivePointerAlignment: false +PointerAlignment: Right +BreakStringLiterals: false +SortIncludes: false +ReflowComments: false diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..7c468e9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,18 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.{c,h}] +indent_style = tab +indent_size = 8 + +[CMakeLists.txt] +indent_style = spaces +indent_size = 4 + +[*.py] +indent_style = spaces +indent_size = 4 + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4ed06a7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +!.clang-format +*.o +*.a +*.pyc +*.pyo +*~ +\#* +src/zmap +src/zopt.h +src/zopt.c +src/zbopt.h +src/zbopt.c +src/zitopt.c +src/zitopt.h +src/topt.c +src/topt.h +src/ztopt.c +src/ztopt.h +src/*.ggo +src/zblocklist +src/ztee +src/ziterate +lexer.c +lexer.h +parser.c +parser.h +*.deb +!.gitignore +!.travis.yml +cmake_installer.sh +CMakeFiles +*.cmake +Makefile +CMakeCache.txt +install_manifest.txt +_CPack_Packages/* +src/ztests +*.DS_Store diff --git a/10gigE.md b/10gigE.md new file mode 100644 index 0000000..0f9219c --- /dev/null +++ b/10gigE.md @@ -0,0 +1,57 @@ +10GigE (Zippier) ZMap +=========== + +It is possible to build ZMap to run at 95% of 10 GigE linespeed, sending over 14 +million packets per second. This requires a compatible Intel 10 Gbps Ethernet +NIC and Linux. + +### Prerequisites + + 0. A working ZMap development environment (see [INSTALL.md](install.md)) + 1. A [PF_RING ZC](http://www.ntop.org/products/pf_ring/pf_ring-zc-zero-copy/) + license from ntop. + 2. PF_RING ZC headers and kernel module + 3. A 10 Gbps NIC with compatible "PF_RING-aware" drivers + 4. A Linux (not BSD or Mac) installation + 5. For best results, a computer with at least 8 *physical* cores on the same + NUMA node. + 6. libnuma (`sudo apt-get install libnuma-dev`) + +### Building + +Most build errors are due to incorrectly building or installing PF_RING. Make +sure you have build the drivers, the kernel module, and the userland library, as +well as install the headers and kernel module to the correct locations. + +The PF_RING `make install` command might not copy `pfring_zc.h` to +`/usr/include`, in which case manually install the file and set permissions +correctly. + +To build navigate to the root of the repository and run: + +``` +$ cmake -DWITH_PFRING=ON -DENABLE_DEVELOPMENT=OFF . +$ make +``` + +### Running + +You'll have to carefully select the number of threads to use, as well as specify +as zero-copy interface, e.g. `zc:eth1`. Use the `--cores` option to pick which +cores to pin to. Make sure to pin to different physical cores, and note that +some machines interleave physical and "virtual" cores. +``` +$ sudo ./src/zmap -p 80 -i zc:eth7 -o output.csv -T 5 +``` + +### Considerations + +DO NOT TAKE THIS LIGHTLY! + +Running ZMap at 10Gbps hits every /16 on the Internet over 200 times a second. +Even if you have a large source IP range to scan from, it's very obvious that +you're scanning. As always, follow scanning best practices, honor blocklist +requests, and signal benign/research intent via domain names and websites on +your scan IPs. + +Remember, you're sending a lot of traffic. diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..43074de --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,168 @@ +cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) +project(ZMAP C) +set(ZMAP_VERSION DEVELOPMENT) # Change DEVELOPMENT to version number for release + +option(ENABLE_DEVELOPMENT "Enable development specific compiler and linker flags" OFF) +option(ENABLE_LOG_TRACE "Enable log trace messages" OFF) +option(RESPECT_INSTALL_PREFIX_CONFIG "Respect CMAKE_INSTALL_PREFIX for /etc" OFF) +option(WITH_WERROR "Build with -Werror" OFF) +option(WITH_PFRING "Build with PF_RING ZC for send (10 GigE)" OFF) +option(FORCE_CONF_INSTALL "Overwrites existing configuration files at install" OFF) + +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(USING_CLANG "YES") +else() + set(USING_GCC "YES") +endif() + +if("${CMAKE_SYSTEM_NAME}" MATCHES "FreeBSD" OR "${CMAKE_SYSTEM_NAME}" MATCHES "NetBSD" OR "${CMAKE_SYSTEM_NAME}" MATCHES "DragonFly") + set(BSD "YES") +endif() + +if("${CMAKE_SYSTEM_NAME}" MATCHES "NetBSD") + set(NetBSD "YES") +endif() + +# Hardening and warnings for building with gcc +# Maybe add -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations +set(GCCWARNINGS + "-Wall -Wformat=2 -Wno-format-nonliteral" + "-pedantic -fno-strict-aliasing" + "-Wextra" + "-Wfloat-equal -Wundef -Wwrite-strings -Wredundant-decls" + "-Wnested-externs -Wbad-function-cast -Winit-self" + "-Wmissing-noreturn" + "-Wstack-protector" +) + +# Fix line breaks +string(REPLACE ";" " " GCCWARNINGS "${GCCWARNINGS}") + +if(WITH_WERROR) + set(GCCWARNINGS "${GCCWARNINGS} -Werror") +endif() + +if(ENABLE_DEVELOPMENT) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ggdb") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -g") +else() + # Hardening and optimizations for building with gcc + set(GCCHARDENING "-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all -fwrapv -fPIC --param ssp-buffer-size=1") + if(NOT APPLE AND NOT BSD) + set(LDHARDENING "-z relro -z now") + else() + set(LDHARDENING "") + endif() + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GCCHARDENING} -O2") + set(CMAKE_EXE_LINKER_FLAGS "${LDHARDENING} ${CMAKE_EXE_LINKER_FLAGS}") +endif() + +if(ENABLE_LOG_TRACE) + add_definitions("-DDEBUG") +endif() + + +set(CMAKE_C_FLAGS "${GCCWARNINGS} ${CMAKE_C_FLAGS}") + + +include(FindPkgConfig) +pkg_check_modules(JSON json-c) +if(JSON_FOUND) + include_directories(${JSON_INCLUDE_DIRS}) +else() + message(FATAL_ERROR "Did not find libjson") +endif() +string(REPLACE ";" " " JSON_CFLAGS "${JSON_CFLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${JSON_CFLAGS}") + +if(WITH_PFRING) + add_definitions("-DPFRING") + set(PFRING_LIBRARIES pfring rt numa) +endif() + +set(JUDY_LIBRARIES "Judy") + +# Standard FLAGS +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99") +if(NOT APPLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread") +endif() + +# Set up OS-specific include directories +if(APPLE) + if(EXISTS /opt/local/include) + include_directories(/opt/local/include) + endif() + if(EXISTS /opt/local/lib) + link_directories(/opt/local/lib) + endif() + if(EXISTS /usr/local/include) + include_directories(/usr/local/include) + endif() + if(EXISTS /usr/local/lib) + link_directories(/usr/local/lib) + endif() + if(EXISTS /opt/homebrew) + include_directories(/opt/homebrew/include) + link_directories(/opt/homebrew/lib) + endif() +endif() + +if(BSD) + include_directories(/usr/local/include) + link_directories(/usr/local/lib) +endif() + +if(NetBSD) + include_directories(/usr/pkg/include) + link_directories(/usr/pkg/lib) +endif() + +add_subdirectory(lib) +add_subdirectory(src) + +# Install conf files +if(RESPECT_INSTALL_PREFIX_CONFIG) + set(CONFIG_DESTINATION "etc/zmap") +else() + set(CONFIG_DESTINATION "/etc/zmap") +endif() + +FILE(GLOB CONF_FILES "${PROJECT_SOURCE_DIR}/conf/*") + +message(STATUS "Default ZMap configuration file location is /etc/zmap") +foreach(EACH_CONF ${CONF_FILES}) + get_filename_component(CONF_BASENAME ${EACH_CONF} NAME) + message(STATUS "Checking if ${CONF_BASENAME} exists there...") + if(NOT EXISTS "${CONFIG_DESTINATION}/${CONF_BASENAME}") + install(FILES ${EACH_CONF} DESTINATION ${CONFIG_DESTINATION}) + elseif(FORCE_CONF_INSTALL) + message(WARNING "FORCE_CONF_INSTALL will overwrite any existing configuration files") + install(FILES ${EACH_CONF} DESTINATION ${CONFIG_DESTINATION}) + else() + message(WARNING "Existing configuration file detected at /etc/zmap/${CONF_BASENAME}, ${CONF_BASENAME} from sources will NOT be installed. Please check and install manually!") + endif() +endforeach() + +# Allow Debian Packaging +include(InstallRequiredSystemLibraries) + +set(CPACK_SET_DESTDIR "on") +set(CPACK_PACKAGING_INSTALL_PREFIX "/tmp") +set(CPACK_GENERATOR "DEB") + +set(CPACK_DEBIAN_PACKAGE_VERSION ${ZMAP_VERSION}) +set(CPACK_DEBIAN_PACKAGE_PRIORITY "optional") +set(CPACK_DEBIAN_PACKAGE_SECTION "network") +set(CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgmp10, libpcap0.8, libjson-c-dev") + +set(CPACK_PACKAGE_DESCRIPTION "Internet-scale network scanner") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "ZMap is an open source network scanner that enables researchers to easily perform Internet-wide network studies. With a single machine and a well provisioned network uplink, ZMap is capable of performing a complete scan of the IPv4 address space in under five minutes, approaching the theoretical limit of gigabit Ethernet. ZMap can be used to study protocol adoption over time, monitor service availability, and help us better understand large systems distributed across the Internet.") +set(CPACK_PACKAGE_CONTACT "Zakir Durumeric ") +set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}_${VERSION}_${CPACK_DEBIAN_ARCHITECTURE}") + +set(CPACK_COMPONENTS_ALL Libraries ApplicationData) + +include(CPack) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ef91264 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,36 @@ +Contributing to ZMap +==================== + +ZMap accepts contributions in the form of issues and pull requests. In either +case, before posting please [search](https://github.com/zmap/zmap/issues) to see +if your change or bug report has been addressed previously. + +[INSTALL](INSTALL.md#building-from-source) provides guidance on building ZMap +from source. + +Developing +---------- + +- ZMap code follows the [Linux kernel style guide][kernelguide]. We maintain [a + configuration file](/.clang-format) for `clang-format` that applies this + style. You can use the [format.sh](/format.sh) script to apply this style. + +- Before submitting a PR, please rebase/squash your commits down to a single + commit. Follow these [commit message guidelines][guidelines], especially with + regard to formatting. + +Reviewing +--------- + +- All commits must be reviewed in the form of a pull request by a ZMap + maintainer. This usually means @zakird or @dadrian (or both). + +- All pull-requests should be squash-merged into master. + +- When squash-merging, put the PR number in the commit title. GitHub does this + automatically in the web interface. Condense the commit messages down to a + single message; often this can just be the commit message from the first + commit in a PR. Follow the commit formatting guidelines [here][guidelines]. + +[kernelguide]: https://www.kernel.org/doc/Documentation/process/coding-style.rst +[guidelines]: https://github.com/torvalds/subsurface-for-dirk/blob/master/README#L92 diff --git a/checkFormat.sh b/checkFormat.sh new file mode 100755 index 0000000..566bb48 --- /dev/null +++ b/checkFormat.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +CLANG_FORMAT=clang-format-6.0 + +files_to_lint=$(find ./src ./lib -type f -name '*.c' -or -name '*.h') + +fail=0 +for f in ${files_to_lint}; do + d="$(diff -u "$f" <($CLANG_FORMAT -style=file "$f") || true)" + if ! [ -z "$d" ]; then + printf "The file %s is not compliant with the coding style:\n%s\n" "$f" "$d" + fail=1 + fi +done + +if [ "$fail" -eq "1" ]; then + if [ ! -z $ZMAP_ENFORCE_FORMAT ]; then + exit 1 + fi +fi diff --git a/conf/blocklist.conf b/conf/blocklist.conf new file mode 100644 index 0000000..018b0f4 --- /dev/null +++ b/conf/blocklist.conf @@ -0,0 +1,25 @@ +# From IANA IPv4 Special-Purpose Address Registry +# http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +# Updated 2013-05-22 + +0.0.0.0/8 # RFC1122: "This host on this network" +10.0.0.0/8 # RFC1918: Private-Use +100.64.0.0/10 # RFC6598: Shared Address Space +127.0.0.0/8 # RFC1122: Loopback +169.254.0.0/16 # RFC3927: Link Local +172.16.0.0/12 # RFC1918: Private-Use +192.0.0.0/24 # RFC6890: IETF Protocol Assignments +192.0.2.0/24 # RFC5737: Documentation (TEST-NET-1) +192.88.99.0/24 # RFC3068: 6to4 Relay Anycast +192.168.0.0/16 # RFC1918: Private-Use +198.18.0.0/15 # RFC2544: Benchmarking +198.51.100.0/24 # RFC5737: Documentation (TEST-NET-2) +203.0.113.0/24 # RFC5737: Documentation (TEST-NET-3) +240.0.0.0/4 # RFC1112: Reserved +255.255.255.255/32 # RFC0919: Limited Broadcast + +# From IANA Multicast Address Space Registry +# http://www.iana.org/assignments/multicast-addresses/multicast-addresses.xhtml +# Updated 2013-06-25 + +224.0.0.0/4 # RFC5771: Multicast/Reserved diff --git a/conf/zmap.conf b/conf/zmap.conf new file mode 100644 index 0000000..817c184 --- /dev/null +++ b/conf/zmap.conf @@ -0,0 +1,22 @@ +### Probe Module to use +#probe-module tcp_synscan + + +### Destination port to scan +#target-port 443 + +### Scan rate in packets/sec +#rate 10000 + +### Scan rate in bandwidth (bits/sec); overrides `rate` +#bandwidth 1M # 1mbps + + +### Blocklist file to use. We encourage you to exclude +### RFC1918, IANA reserved, and multicast networks, +### in addition to those who have opted out of your +### network scans. +blocklist-file "/etc/zmap/blocklist.conf" + +### Optionally print a summary at the end +#summary diff --git a/examples/forge-socket/README b/examples/forge-socket/README new file mode 100644 index 0000000..05282ab --- /dev/null +++ b/examples/forge-socket/README @@ -0,0 +1,4 @@ +Forge Socket +============ + +Forge Socket is now maintained at https://github.com/ewust/forge_socket. diff --git a/examples/probe-modules/module_tcp_cisco_backdoor.c b/examples/probe-modules/module_tcp_cisco_backdoor.c new file mode 100644 index 0000000..9cd7bde --- /dev/null +++ b/examples/probe-modules/module_tcp_cisco_backdoor.c @@ -0,0 +1,203 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" + +probe_module_t module_tcp_synscan; +static uint32_t num_ports; + +static int synscan_global_initialize(struct state_conf *state) +{ + num_ports = state->source_port_last - state->source_port_first + 1; + return EXIT_SUCCESS; +} + +static int synscan_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + port_h_t dst_port, + __attribute__((unused)) void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + sizeof(struct tcphdr)); + make_ip_header(ip_header, IPPROTO_TCP, len); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + make_tcp_header(tcp_header, dst_port, TH_SYN); + return EXIT_SUCCESS; +} + +// instead of settings sequence number to be random for validation +// let's instead set to something static so that we can easily +// set acknowledgement number. I don't know how integer overflow +// is going to act in this. +// uint32_t tcp_seq = validation[0]; +// From Mandiant +// 1. To initiate the process, a uniquely crafted TCP SYN packet is sent +// to port 80 of the “implanted” router. It is important to note that +// the difference between the sequence and acknowledgment numbers must +// be set to 0xC123D. Also the ACK number doesn’t need to be zero. + +#define BACKDOOR_SEQ 0x3D120C00 +//#define BACKDOOR_SEQ 0x000C123D // wrong byte order +#define BACKDOOR_ACK 0x0 +#define EXPECTED_RESPONSE_SEQ 0 +//#define EXPECTED_RESPONSE_ACK 0x000C123E // wrong byte order +#define EXPECTED_RESPONSE_ACK 0x3E120C00 + +static int synscan_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + tcp_header->th_sport = + htons(get_src_port(num_ports, probe_num, validation)); + tcp_header->th_seq = BACKDOOR_SEQ; + tcp_header->th_ack = BACKDOOR_ACK; + tcp_header->th_sum = 0; + tcp_header->th_sum = + tcp_checksum(sizeof(struct tcphdr), ip_header->ip_src.s_addr, + ip_header->ip_dst.s_addr, tcp_header); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + return EXIT_SUCCESS; +} + +static void synscan_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct tcphdr *tcph = (struct tcphdr *)&iph[1]; + fprintf(fp, + "tcp { source: %u | dest: %u | seq: %u | checksum: %#04X }\n", + ntohs(tcph->th_sport), ntohs(tcph->th_dport), + ntohl(tcph->th_seq), ntohs(tcph->th_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, "------------------------------------------------------\n"); +} + +static int synscan_validate_packet(const struct ip *ip_hdr, uint32_t len, + __attribute__((unused)) uint32_t *src_ip, + uint32_t *validation) +{ + if (ip_hdr->ip_p != IPPROTO_TCP) { + return 0; + } + if ((4 * ip_hdr->ip_hl + sizeof(struct tcphdr)) > len) { + // buffer not large enough to contain expected tcp header + return 0; + } + struct tcphdr *tcp = + (struct tcphdr *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + uint16_t sport = tcp->th_sport; + uint16_t dport = tcp->th_dport; + // validate source port + if (ntohs(sport) != zconf.target_port) { + return 0; + } + // validate destination port + if (!check_dst_port(ntohs(dport), num_ports, validation)) { + return 0; + } + // DO NOT validate ack number as this is currently statically set + // validate tcp acknowledgement number + // if (htonl(tcp->th_ack) != htonl(validation[0])+1) { + // return 0; + //} + return 1; +} + +static void synscan_process_packet(const u_char *packet, uint32_t len, + fieldset_t *fs, + __attribute__((unused)) uint32_t *validation, + __attribute__((unused)) struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + struct tcphdr *tcp = + (struct tcphdr *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + + fs_add_uint64(fs, "sport", (uint64_t)ntohs(tcp->th_sport)); + fs_add_uint64(fs, "dport", (uint64_t)ntohs(tcp->th_dport)); + fs_add_uint64(fs, "seqnum", (uint64_t)ntohl(tcp->th_seq)); + fs_add_uint64(fs, "acknum", (uint64_t)ntohl(tcp->th_ack)); + fs_add_uint64(fs, "window", (uint64_t)ntohs(tcp->th_win)); + fs_add_uint64(fs, "urgentptr", (uint64_t)ntohs(tcp->th_urp)); + fs_add_uint64(fs, "flags", (uint64_t)ntohs(tcp->th_flags)); + fs_add_binary(fs, "raw", len, (void *)packet, 0); + + if (tcp->th_flags & TH_RST) { // RST packet + fs_add_string(fs, "classification", (char *)"rst", 0); + fs_add_bool(fs, "success", 0); + } else if (tcp->th_seq == EXPECTED_RESPONSE_SEQ && tcp->th_urp) { + fs_add_string(fs, "classification", (char *)"backdoor", 0); + fs_add_bool(fs, "success", 1); + } else { // SYNACK packet + fs_add_string(fs, "classification", (char *)"synack", 0); + fs_add_bool(fs, "success", 1); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window"}, + {.name = "urgentptr", .type = "int", .desc = "Urgent POinter"}, + {.name = "flags", .type = "int", .desc = "tcp flags"}, + {.name = "raw", .type = "binary", .desc = "raw packet"}, + {.name = "classification", + .type = "string", + .desc = "packet classification"}, + {.name = "success", + .type = "bool", + .desc = "is response considered success"}}; + +probe_module_t module_tcp_cisco_backdoor = { + .name = "tcp_cisco_backdoor", + .packet_length = 54, + .pcap_filter = "tcp && tcp[13] & 4 != 0 || tcp[13] == 18", + .pcap_snaplen = 256, + .port_args = 1, + .global_initialize = &synscan_global_initialize, + .thread_initialize = &synscan_init_perthread, + .make_packet = &synscan_make_packet, + .print_packet = &synscan_print_packet, + .process_packet = &synscan_process_packet, + .validate_packet = &synscan_validate_packet, + .close = NULL, + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response.", + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = 10}; diff --git a/examples/udp-probes/README b/examples/udp-probes/README new file mode 100644 index 0000000..b925b51 --- /dev/null +++ b/examples/udp-probes/README @@ -0,0 +1,51 @@ + +UDP Data Probes +====== + +This directory contains a set of data files that can be used with the UDP probe module. + + +USING: +----- + +$ zmap -M udp -p 137 --probe-args=file:examples/udp-probes/netbios_137.pkt + + +PROBES: +----- + +citrix_1604.pkt This probe triggers a response from Citrix application discovery services on UDP port 1604 +db2disco_523.pkt This probe triggers a response from IBM DB2 discovery services on UDP port 523 +digi1_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (default magic) +digi2_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (devkit magic) +digi3_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (oem magic) +dns_53.pkt This probe queries for the DNS vendor and version using the BIND version TXT record over UDP port 53 +dns_53_queryAwww.google.it.pkt This probe queries for the domain www.google.it A record over UDP port 53 +dns_53_queryAwww.google.com.pkt This probe queries for the domain www.google.com A record over UDP port 53 +ipmi_623.pkt This probe triggers a Get Channel Authentication reply from IPMI endpoints on UDP port 623 +mdns_5353.pkt This probe triggers a response from mDNS/Avahi/Bonjour discovery services on UDP port 5353 +memcache_11211.pkt This probe triggers a response from memcached on UDP port 11211 (stats items). +mssql_1434.pkt This probe triggers a response from Microsoft SQL Server discovery services on UDP port 1434 +natpmp_5351.pkt This probe triggers a response from NATPMP-enabled devices on UDP port 5351 +netbios_137.pkt This probe triggers a status reply from NetBIOS services on UDP port 137 +ntp_123.pkt This probe triggers a response from NTP services on UDP port 123 +ntp_123_monlist.pkt This probe triggers a response for command "monlist" from NTP services on UDP port 123 +pca_nq_5632.pkt This probe triggers a response from PC Anywhere services on UDP port 5632 (network query) +pca_st_5632.pkt This probe triggers a response from PC Anywhere services on UDP port 5632 (status) +portmap_111.pkt This probe triggers a response from SunRPC portmapper services on UDP port 111 +ripv1_520.pkt This probe triggers a response from the RIPv1 enabled routers/devices on UDP port 520 +sentinel_5093.pkt This probe triggers a response from the Sentinel license manager service on UDP port 5093 +snmp1_161.pkt This probe queries for the system description field of SNMP v1 services using community string public over UDP port 161 +snmp2_161.pkt This probe queries for the system description field of SNMP v2 services using community string public over UDP port 161 +snmp3_161.pkt This probe triggers a response from SNMP v3 services on UDP port 161 +upnp_1900.pkt This probe triggers a response from UPnP SSDP services on UDP port 1900 +wdbrpc_17185.pkt This probe triggers a response from VxWorks WDBRPC services on UDP port 17185 +wsd_3702.pkt This probe triggers a response from WSD/DPWS services on UDP port 3702 +coap_5683.pkt This probe triggers a response from COAP services on UDP port 5683 + +NOTES: +----- + +Most of these probes return useful data in the response. Parsing this data requires capturing the raw output +and decoding this using a protocol-specific dissector. In most cases, Wireshark is capable of decoding these +replies. diff --git a/examples/udp-probes/ard_3283.pkt b/examples/udp-probes/ard_3283.pkt new file mode 100644 index 0000000000000000000000000000000000000000..89f6baf6d4ef6d221eaa1ad854a9904d1faf2e0e GIT binary patch literal 5 McmZP&VPIqi003G582|tP literal 0 HcmV?d00001 diff --git a/examples/udp-probes/bacnet_47808.pkt b/examples/udp-probes/bacnet_47808.pkt new file mode 100644 index 0000000000000000000000000000000000000000..6986139bc6b9289ecf94b9e7b8841b2e7e00246b GIT binary patch literal 18 ZcmZqVVi06xVPL(+!^33%|G%WS4*(z<1cU$p literal 0 HcmV?d00001 diff --git a/examples/udp-probes/bacnet_rpm_47808.pkt b/examples/udp-probes/bacnet_rpm_47808.pkt new file mode 100644 index 0000000000000000000000000000000000000000..041c9568a825a4d27b0d953c35fe256dbab8d2f0 GIT binary patch literal 37 scmZqVVo+sdVPa+E<6&Z8V36bF;gsRj;k4ki9GIDnfaddGBVh9fOF|af+G++P#H1GtT literal 0 HcmV?d00001 diff --git a/examples/udp-probes/db2disco_523.pkt b/examples/udp-probes/db2disco_523.pkt new file mode 100644 index 0000000000000000000000000000000000000000..388374500e3c296faa4f114235de895e16256900 GIT binary patch literal 20 bcmZ>9GIDnfaddGBVh9fOF)%eSFkk=xG}#21 literal 0 HcmV?d00001 diff --git a/examples/udp-probes/digi1_2362.pkt b/examples/udp-probes/digi1_2362.pkt new file mode 100644 index 0000000000000000000000000000000000000000..74f57b88f68d8dd24a9972ffdb95e10e3e0d53d4 GIT binary patch literal 14 RcmZ?qboXRnWMKOb1^^=v2Oj_c literal 0 HcmV?d00001 diff --git a/examples/udp-probes/digi2_2362.pkt b/examples/udp-probes/digi2_2362.pkt new file mode 100644 index 0000000000000000000000000000000000000000..d962606efade4a7fe8103769fcb15c68a1a69a62 GIT binary patch literal 14 RcmZ<>^A2HPWMKOb1^^^t2Ri@& literal 0 HcmV?d00001 diff --git a/examples/udp-probes/digi3_2362.pkt b/examples/udp-probes/digi3_2362.pkt new file mode 100644 index 0000000000000000000000000000000000000000..ffba12522e33d978a04cb02e9f251f8326f381c0 GIT binary patch literal 14 RcmZ>9cL`u%WMKOb1^^=;2O$6e literal 0 HcmV?d00001 diff --git a/examples/udp-probes/dns_53.pkt b/examples/udp-probes/dns_53.pkt new file mode 100644 index 0000000000000000000000000000000000000000..616e17e8c6d512a676c5c60d14091fe3ab95e32f GIT binary patch literal 30 hcmXqc&&a?4L?FN(<{A|2>F>wlOa1OQoB1?>O; literal 0 HcmV?d00001 diff --git a/examples/udp-probes/dns_53_queryAwww.google.it.pkt b/examples/udp-probes/dns_53_queryAwww.google.it.pkt new file mode 100644 index 0000000000000000000000000000000000000000..99238c0f957efc324a7fae5af82f2cb3a527eddd GIT binary patch literal 31 gcmeZfWn^FgA`oCMFE3|H&(BZKNoC3`0SPk#07Rb!9{>OV literal 0 HcmV?d00001 diff --git a/examples/udp-probes/ipmi_623.pkt b/examples/udp-probes/ipmi_623.pkt new file mode 100644 index 0000000000000000000000000000000000000000..025d0fa01a82e5bc97fe5238be7ed732cf43fee4 GIT binary patch literal 23 YcmZQ$_|MJ&1DpyHCmIffO!q)L+@VaAVMC0B6Jn%>V!Z literal 0 HcmV?d00001 diff --git a/examples/udp-probes/ntp_123_monlist.pkt b/examples/udp-probes/ntp_123_monlist.pkt new file mode 100644 index 0000000000000000000000000000000000000000..ba2912d265d286d8348d046996d4eabeb08ec85b GIT binary patch literal 192 OcmWeJOD)i literal 0 HcmV?d00001 diff --git a/examples/udp-probes/openvpn_1194.pkt b/examples/udp-probes/openvpn_1194.pkt new file mode 100644 index 0000000000000000000000000000000000000000..fec8aa2cf47b0239171bcbb6f81c882b86f77d25 GIT binary patch literal 18 RcmcBzf&vB*u-K?}4*(Ch0=57E literal 0 HcmV?d00001 diff --git a/examples/udp-probes/pca_nq_5632.pkt b/examples/udp-probes/pca_nq_5632.pkt new file mode 100644 index 0000000..8d51173 --- /dev/null +++ b/examples/udp-probes/pca_nq_5632.pkt @@ -0,0 +1 @@ +NQ \ No newline at end of file diff --git a/examples/udp-probes/pca_st_5632.pkt b/examples/udp-probes/pca_st_5632.pkt new file mode 100644 index 0000000..86aa1fa --- /dev/null +++ b/examples/udp-probes/pca_st_5632.pkt @@ -0,0 +1 @@ +ST \ No newline at end of file diff --git a/examples/udp-probes/pcanywhere_5632.pkt b/examples/udp-probes/pcanywhere_5632.pkt new file mode 100644 index 0000000000000000000000000000000000000000..a5c6833ceacbcc3c1e87c7b66dc12d48f5f29259 GIT binary patch literal 18 QcmeYZWIzIrU-mBp01)#7IsgCw literal 0 HcmV?d00001 diff --git a/examples/udp-probes/portmap_111.pkt b/examples/udp-probes/portmap_111.pkt new file mode 100644 index 0000000000000000000000000000000000000000..7cfbdb97ae1a834dc9f4631c5fa89086aa700c33 GIT binary patch literal 40 bcmb1I`TrpU1TZl$wk-fMnShuDB7`6Tf>i_4 literal 0 HcmV?d00001 diff --git a/examples/udp-probes/qotd_17.pkt b/examples/udp-probes/qotd_17.pkt new file mode 100644 index 0000000..d3f5a12 --- /dev/null +++ b/examples/udp-probes/qotd_17.pkt @@ -0,0 +1 @@ + diff --git a/examples/udp-probes/rdp_3389.pkt b/examples/udp-probes/rdp_3389.pkt new file mode 100644 index 0000000000000000000000000000000000000000..2423120883e3044ebcb6be6b1d59c188369fef69 GIT binary patch literal 18 RcmZQz0E7P!ih&`74FC(Z0a^e6 literal 0 HcmV?d00001 diff --git a/examples/udp-probes/ripv1_520.pkt b/examples/udp-probes/ripv1_520.pkt new file mode 100644 index 0000000000000000000000000000000000000000..56ea97bb925dcf600a776a1deb157e0612499acf GIT binary patch literal 24 NcmZQ%WWWLh7ytlg022TJ literal 0 HcmV?d00001 diff --git a/examples/udp-probes/sentinel_5093.pkt b/examples/udp-probes/sentinel_5093.pkt new file mode 100644 index 0000000000000000000000000000000000000000..158b3a9dd9dcfc088b7fa278d91ac48eb8605ea3 GIT binary patch literal 6 KcmbR90LWy7VyG~L`u{&v2*|R5$}zzt0r6V})Bpeg literal 0 HcmV?d00001 diff --git a/examples/udp-probes/wsd_3702.pkt b/examples/udp-probes/wsd_3702.pkt new file mode 100644 index 0000000..87704a5 --- /dev/null +++ b/examples/udp-probes/wsd_3702.pkt @@ -0,0 +1,3 @@ + + +urn:schemas-xmlsoap-org:ws:2005:04:discoveryhttp://schemas.xmlsoap.org/ws/2005/04/discovery/Probeurn:uuid:ce04dad0-5d2c-4026-9146-1aabfc1e4111wsdp:Device diff --git a/examples/udp-probes/wsd_malformed_3702.pkt b/examples/udp-probes/wsd_malformed_3702.pkt new file mode 100644 index 0000000..9fd51b3 --- /dev/null +++ b/examples/udp-probes/wsd_malformed_3702.pkt @@ -0,0 +1 @@ +<:/> diff --git a/examples/udp-probes/xdmcp_177.pkt b/examples/udp-probes/xdmcp_177.pkt new file mode 100644 index 0000000000000000000000000000000000000000..96a017a5febf4f8c862591a68ac7841f3f00dc40 GIT binary patch literal 7 OcmZQzWME=oWB>pF7XSqS literal 0 HcmV?d00001 diff --git a/format.sh b/format.sh new file mode 100755 index 0000000..7380192 --- /dev/null +++ b/format.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e +set -o pipefail + +MAJOR_REV=$((clang-format --version | awk '{print $3}' | cut -d '.' -f 1) || echo 0) +if [ $MAJOR_REV -lt 5 ]; then + echo "error: need at least clang-format version 5.x" + exit 1 +fi + +FORMAT_CMD="clang-format -i -style=file" + +# No files passed, format everything +if [ $# -eq 0 ]; then + echo "formatting all C code in src/ and lib/" + find ./src -type f -name '*.c' -exec $FORMAT_CMD {} \; + find ./src -type f -name '*.h' -exec $FORMAT_CMD {} \; + find ./lib -type f -name '*.c' -exec $FORMAT_CMD {} \; + find ./lib -type f -name '*.h' -exec $FORMAT_CMD {} \; + exit 0 +fi + +# File names passed, format only those files +$FORMAT_CMD $@ diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt new file mode 100644 index 0000000..39baf65 --- /dev/null +++ b/lib/CMakeLists.txt @@ -0,0 +1,23 @@ +SET(LIB_SOURCES + blocklist.c + cachehash.c + constraint.c + logger.c + pbm.c + random.c + rijndael-alg-fst.c + xalloc.c + lockfd.c + util.c + queue.c + csv.c +) + +add_library(zmaplib STATIC ${LIB_SOURCES}) + +target_link_libraries( + zmaplib + ${JUDY_LIBRARIES} +) + +target_include_directories (zmaplib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/lib/blocklist.c b/lib/blocklist.c new file mode 100644 index 0000000..9d28653 --- /dev/null +++ b/lib/blocklist.c @@ -0,0 +1,308 @@ +/* + * Blocklist Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "blocklist.h" +#include "constraint.h" +#include "logger.h" +#include "xalloc.h" + +#define ADDR_DISALLOWED 0 +#define ADDR_ALLOWED 1 + +typedef struct bl_linked_list { + bl_cidr_node_t *first; + bl_cidr_node_t *last; + uint32_t len; +} bl_ll_t; + +static constraint_t *constraint = NULL; + +// keep track of the prefixes we've tried to BL/WL +// for logging purposes +static bl_ll_t *blocklisted_cidrs = NULL; +static bl_ll_t *allowlisted_cidrs = NULL; + +void bl_ll_add(bl_ll_t *l, struct in_addr addr, uint16_t p) +{ + assert(l); + bl_cidr_node_t *new = xmalloc(sizeof(bl_cidr_node_t)); + new->next = NULL; + new->ip_address = addr.s_addr; + new->prefix_len = p; + if (!l->first) { + l->first = new; + } else { + l->last->next = new; + } + l->last = new; + l->len++; +} + +bl_cidr_node_t *get_blocklisted_cidrs(void) { return blocklisted_cidrs->first; } + +bl_cidr_node_t *get_allowlisted_cidrs(void) { return allowlisted_cidrs->first; } + +uint32_t blocklist_lookup_index(uint64_t index) +{ + return ntohl(constraint_lookup_index(constraint, index, ADDR_ALLOWED)); +} + +// check whether a single IP address is allowed to be scanned. +// 1 => is allowed +// 0 => is not allowed +int blocklist_is_allowed(uint32_t s_addr) +{ + return constraint_lookup_ip(constraint, ntohl(s_addr)) == ADDR_ALLOWED; +} + +static void _add_constraint(struct in_addr addr, int prefix_len, int value) +{ + constraint_set(constraint, ntohl(addr.s_addr), prefix_len, value); + if (value == ADDR_ALLOWED) { + bl_ll_add(allowlisted_cidrs, addr, prefix_len); + } else if (value == ADDR_DISALLOWED) { + bl_ll_add(blocklisted_cidrs, addr, prefix_len); + } else { + log_fatal("blocklist", + "unknown type of blocklist operation specified"); + } +} + +// blocklist a CIDR network allocation +// e.g. blocklist_add("128.255.134.0", 24) +void blocklist_prefix(char *ip, int prefix_len) +{ + struct in_addr addr; + addr.s_addr = inet_addr(ip); + _add_constraint(addr, prefix_len, ADDR_DISALLOWED); +} + +// allowlist a CIDR network allocation +void allowlist_prefix(char *ip, int prefix_len) +{ + struct in_addr addr; + addr.s_addr = inet_addr(ip); + _add_constraint(addr, prefix_len, ADDR_ALLOWED); +} + +static int is_ip_ipv6(char *ip) { + // don't modify the input string + char *new_str = strdup(ip); + // check if there's a subnet mask_char + char *mask_char = strchr(new_str, '/'); + if (mask_char != NULL) { + // set mask_char char to NULL char, so we can check if subnet is valid IPv6 + *mask_char = '\0'; + } + // attempt conversion of IP into IPv6 struct to check if IP is an IPv6 address + struct in6_addr ipv6_addr; + if (inet_pton(AF_INET6, new_str, &ipv6_addr) == 1) { + free(new_str); + return true; + } + free(new_str); + return false; +} + +static int init_from_string(char *ip, int value) +{ + if (is_ip_ipv6(ip)) { + log_debug("constraint", "ignoring IPv6 IP/subnet: %s", ip); + return 0; + } + int prefix_len = 32; + char *slash = strchr(ip, '/'); + if (slash) { // split apart network and prefix length + *slash = '\0'; + char *end; + char *len = slash + 1; + errno = 0; + prefix_len = strtol(len, &end, 10); + if (end == len || errno != 0 || prefix_len < 0 || + prefix_len > 32) { + log_fatal("constraint", + "'%s' is not a valid IPv4 prefix length", len); + return -1; + } + } + struct in_addr addr; + int ret = -1; + if (inet_aton(ip, &addr) == 0) { + // Not an IP and not a CIDR block, try dns resolution + struct addrinfo hint, *res; + memset(&hint, 0, sizeof(hint)); + hint.ai_family = PF_INET; + int r = getaddrinfo(ip, NULL, &hint, &res); + if (r) { + log_error("constraint", + "'%s' is not a valid IP " + "address or hostname", + ip); + return -1; + } + // Got some addrinfo, let's see what happens + for (struct addrinfo *aip = res; aip; aip = aip->ai_next) { + if (aip->ai_family != AF_INET) { + continue; + } + struct sockaddr_in *sa = + (struct sockaddr_in *)aip->ai_addr; + memcpy(&addr, &sa->sin_addr, sizeof(addr)); + log_debug("constraint", "%s retrieved by hostname", + inet_ntoa(addr)); + ret = 0; + _add_constraint(addr, prefix_len, value); + } + } else { + _add_constraint(addr, prefix_len, value); + return 0; + } + return ret; +} + +static int init_from_file(char *file, const char *name, int value, + int ignore_invalid_hosts) +{ + FILE *fp; + char line[1000]; + + fp = fopen(file, "r"); + if (fp == NULL) { + log_fatal(name, "unable to open %s file: %s: %s", name, file, + strerror(errno)); + } + + while (fgets(line, sizeof(line), fp) != NULL) { + char *comment = strchr(line, '#'); + if (comment) { + *comment = '\0'; + } + // hostnames can be up to 255 bytes + char ip[256]; + if ((sscanf(line, "%255s", ip)) == EOF) { + continue; + } + if (init_from_string(ip, value)) { + if (!ignore_invalid_hosts) { + log_fatal(name, "unable to parse %s file: %s", + name, file); + } + } + } + fclose(fp); + + return 0; +} + +static void init_from_array(char **cidrs, size_t len, int value, + int ignore_invalid_hosts) +{ + for (int i = 0; i < (int)len; i++) { + int ret = init_from_string(cidrs[i], value); + if (ret && !ignore_invalid_hosts) { + log_fatal("constraint", + "Unable to init from CIDR list"); + } + } +} + +uint64_t blocklist_count_allowed(void) +{ + assert(constraint); + return constraint_count_ips(constraint, ADDR_ALLOWED); +} + +uint64_t blocklist_count_not_allowed(void) +{ + assert(constraint); + return constraint_count_ips(constraint, ADDR_DISALLOWED); +} + +// network order +uint32_t blocklist_ip_to_index(uint32_t ip) +{ + assert(constraint); + uint32_t ip_hostorder = ntohl(ip); + return constraint_lookup_ip(constraint, ip_hostorder); +} + +// Initialize address constraints from allowlist and blocklist files. +// Either can be set to NULL to omit. +int blocklist_init(char *allowlist_filename, char *blocklist_filename, + char **allowlist_entries, size_t allowlist_entries_len, + char **blocklist_entries, size_t blocklist_entries_len, + int ignore_invalid_hosts) +{ + assert(!constraint); + + blocklisted_cidrs = xcalloc(1, sizeof(bl_ll_t)); + allowlisted_cidrs = xcalloc(1, sizeof(bl_ll_t)); + + if (allowlist_filename && allowlist_entries) { + log_warn("allowlist", + "both a allowlist file and destination addresses " + "were specified. The union of these two sources " + "will be utilized."); + } + if (allowlist_filename || allowlist_entries_len > 0) { + // using a allowlist, so default to allowing nothing + constraint = constraint_init(ADDR_DISALLOWED); + log_debug("constraint", "blocklisting 0.0.0.0/0"); + if (allowlist_filename) { + init_from_file(allowlist_filename, "allowlist", + ADDR_ALLOWED, ignore_invalid_hosts); + } + if (allowlist_entries) { + init_from_array(allowlist_entries, + allowlist_entries_len, ADDR_ALLOWED, + ignore_invalid_hosts); + } + } else { + // no allowlist, so default to allowing everything + log_debug("blocklist", + "no allowlist file or allowlist entries provided"); + constraint = constraint_init(ADDR_ALLOWED); + } + if (blocklist_filename) { + init_from_file(blocklist_filename, "blocklist", ADDR_DISALLOWED, + ignore_invalid_hosts); + } + if (blocklist_entries) { + init_from_array(blocklist_entries, blocklist_entries_len, + ADDR_DISALLOWED, ignore_invalid_hosts); + } + init_from_string(strdup("0.0.0.0"), ADDR_DISALLOWED); + constraint_paint_value(constraint, ADDR_ALLOWED); + uint64_t allowed = blocklist_count_allowed(); + log_debug("constraint", + "%lu addresses (%0.0f%% of address " + "space) can be scanned", + allowed, allowed * 100. / ((long long int)1 << 32)); + if (!allowed) { + log_error("blocklist", + "no addresses are eligible to be scanned in the " + "current configuration. This may be because the " + "blocklist being used by ZMap (%s) prevents " + "any addresses from receiving probe packets.", + blocklist_filename); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/lib/blocklist.h b/lib/blocklist.h new file mode 100644 index 0000000..0fac9f6 --- /dev/null +++ b/lib/blocklist.h @@ -0,0 +1,50 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#ifndef BLACKLIST_H +#define BLACKLIST_H + +typedef struct bl_cidr_node { + uint32_t ip_address; + int prefix_len; + struct bl_cidr_node *next; +} bl_cidr_node_t; + +uint32_t blocklist_lookup_index(uint64_t index); + +int blocklist_is_allowed(uint32_t s_addr); + +void blocklist_prefix(char *ip, int prefix_len); + +void allowlist_prefix(char *ip, int prefix_len); + +int blocklist_init(char *allowlist, char *blocklist, char **allowlist_entries, + size_t allowlist_entries_len, char **blocklist_entries, + size_t blocklist_entries_len, int ignore_invalid_hosts); + +uint64_t blocklist_count_allowed(void); + +uint64_t blocklist_count_not_allowed(void); + +uint32_t blocklist_ip_to_index(uint32_t ip); + +bl_cidr_node_t *get_blocklisted_cidrs(void); +bl_cidr_node_t *get_allowlisted_cidrs(void); + +#endif diff --git a/lib/cachehash.c b/lib/cachehash.c new file mode 100644 index 0000000..a575fcf --- /dev/null +++ b/lib/cachehash.c @@ -0,0 +1,275 @@ +/* + * CacheHash Copyright 2014 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "cachehash.h" + +#include +#include +#include +#include +#include +#include + +#include + +#define EVICTION_NEEDED 1 +#define EVICTION_UNNEC 0 + +// doubly-linked-list node +typedef struct node { + struct node *next; + struct node *prev; + void *key; + size_t keylen; + void *data; +} node_t; + +// data structure that contains the enterity of a linked list +// we typedef this as cachehash in cachehash.h s.t. external interface is clean +struct cachehash_s { + Pvoid_t judy; + void *malloced; + node_t *start; + node_t *end; + node_t *curr_end; + size_t maxsize; + size_t currsize; + cachehash_process_cb *evict_cb; +}; + +cachehash *cachehash_init(size_t maxitems, cachehash_process_cb *cb) +{ + assert(maxitems > 0); + cachehash *retv = malloc(sizeof(cachehash)); + assert(retv); + memset(retv, 0, sizeof(cachehash)); + // allocate nodes all at once to avoid fragmented memory + node_t *nodes = calloc(maxitems, sizeof(node_t)); + retv->malloced = nodes; + assert(nodes); + retv->start = nodes; + retv->curr_end = nodes; + retv->end = &nodes[maxitems - 1]; + retv->maxsize = maxitems; + retv->currsize = 0; + // initial node + nodes[0].next = &nodes[1]; + // middle nodes + for (unsigned int i = 1; i < maxitems - 1; i++) { + nodes[i].prev = &nodes[i - 1]; + nodes[i].next = &nodes[i + 1]; + } + // last node + nodes[maxitems - 1].prev = &nodes[maxitems - 2]; + retv->evict_cb = cb; + return retv; +} + +void cachehash_set_evict_cb(cachehash *ch, cachehash_process_cb *cb) +{ + ch->evict_cb = cb; +} + +// is the hashcache full? +static inline int eviction_needed(cachehash *ch) +{ + assert(ch); + return ch->currsize == ch->maxsize; +} + +// completely evict the LRU object +// does not cb to user w/ object +static inline void *evict(cachehash *ch) +{ + + assert(ch); + node_t *last = ch->end; + // remove item from judy array + int rc; + JHSD(rc, ch->judy, last->key, last->keylen); + // we should never end up with something in the linked list + // that's not in the judy array. + assert(rc); + // reset linked list node + void *retv = last->data; + last->data = NULL; + free(last->key); + last->key = NULL; + last->keylen = 0; + ch->currsize--; + ch->curr_end = ch->end; + return retv; +} + +static inline void use(cachehash *ch, node_t *n) +{ + + assert(ch); + assert(n); + //if first node, nothing to do and return + if (n == ch->start) { + return; + } + // remove from current spot in linked list + node_t *prev = n->prev; + n->prev->next = n->next; + // if last node then no next, but must update LL + if (n->next) { + n->next->prev = prev; + } else { + ch->end = prev; + } + // front of list + n->next = ch->start; + ch->start->prev = n; + ch->start = n; + n->prev = NULL; +} + +static inline node_t *judy_get(cachehash *ch, void *key, size_t keylen) +{ + assert(ch); + assert(key); + assert(keylen); + Word_t *v_; + JHSG(v_, ch->judy, key, keylen); + if (!v_) { + return NULL; + } + return (node_t *)*v_; +} + +void *cachehash_has(cachehash *ch, const void *key, size_t keylen) +{ + assert(ch); + assert(key); + assert(keylen); + node_t *n = judy_get(ch, (void *)key, keylen); + if (n) { + return n->data; + } else { + return NULL; + } +} + +void *cachehash_get(cachehash *ch, const void *key, size_t keylen) +{ + assert(ch); + assert(key); + assert(keylen); + + node_t *n = judy_get(ch, (void *)key, keylen); + if (n) { + use(ch, n); + return n->data; + } else { + return NULL; + } +} + +void *cachehash_evict_if_full(cachehash *ch) +{ + assert(ch); + if (eviction_needed(ch) == EVICTION_UNNEC) { + return NULL; + } + return evict(ch); +} + +void cachehash_put(cachehash *ch, const void *key, size_t keylen, void *value) +{ + assert(ch); + assert(key); + assert(keylen); + + void *evicted = cachehash_evict_if_full(ch); + if (evicted && ch->evict_cb) { + ch->evict_cb(evicted); + ch->curr_end = ch->end; + } + // create new node + + node_t *n; + void *newkey = malloc(keylen); + n = ch->curr_end; + memcpy(newkey, key, keylen); + + n->key = newkey; + n->keylen = keylen; + n->data = value; + //n->prev = ch->curr_end->prev; + //n->next = ch->curr_end->next; + + if (ch->curr_end != ch->end) { + ch->curr_end = ch->curr_end->next; + ch->curr_end->prev = n; + } + + use(ch, n); + ch->currsize++; + // add to judy array + Word_t *v_; + JHSI(v_, ch->judy, (void *)key, keylen); + // key should not already be in hash table + assert(!*v_); + *v_ = (Word_t)n; +} + +// print out entire state. +void cachehash_debug_dump(cachehash *ch) +{ + printf("Statistics:\n"); + printf("\tcurrent size: %lu\n", ch->currsize); + printf("\tmaximum size: %lu\n", ch->maxsize); + printf("\n"); + printf("Linked List:\n"); + size_t i = 0; + node_t *n = ch->start; + + do { + if (n->key) { + printf("\t%lu: %s -> %s\n", i++, (char *)n->key, + (char *)n->data); + } else { + printf("\t%lu: EMPTY\n", i++); + } + n = n->next; + } while (n); +} + +void cachehash_free(cachehash *ch, cachehash_process_cb *cb) +{ + assert(ch); + int rc; + JHSFA(rc, ch->judy); + node_t *n = ch->start; + do { + if (n->key) { + free(n->key); + if (cb) { + cb(n->data); + } + } + n = n->next; + } while (n); + free(ch->malloced); + free(ch); +} + +void cachehash_iter(cachehash *ch, cachehash_process_cb *cb) +{ + node_t *n = ch->start; + do { + if (n->key) { + cb(n->data); + } else { + break; + } + n = n->next; + } while (n); +} diff --git a/lib/cachehash.h b/lib/cachehash.h new file mode 100644 index 0000000..94a2073 --- /dev/null +++ b/lib/cachehash.h @@ -0,0 +1,46 @@ +/* + * CacheHash Copyright 2014 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef CACHEHASH_H +#define CACHEHASH_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct cachehash_s cachehash; +// function defintion for cachehash callbacks +typedef void(cachehash_process_cb)(void *data); +// initialize new cache hash +cachehash *cachehash_init(size_t maxitems, cachehash_process_cb *cb); +// return item from cachehash without changing its location in LL +void *cachehash_has(cachehash *ch, const void *key, size_t keylen); +// return item from cachehash and move to front +void *cachehash_get(cachehash *ch, const void *key, size_t keylen); +// add item to the cachehash +void cachehash_put(cachehash *ch, const void *key, size_t keylen, void *value); +// free memory used by a cachehash. unusable until new initialization +void cachehash_free(cachehash *ch, cachehash_process_cb *cb); +// evict the LRU if the cachehash is full +void *cachehash_evict_if_full(cachehash *ch); +// iterate over all values in the cache hash in MRU -> LRU order +void cachehash_iter(cachehash *ch, cachehash_process_cb *cb); +// print out hash cache to stdout assuming that all keys and values +// are null-terminated ASCII strings +void cachehash_debug_dump(cachehash *ch); +// change the callback function for the cachehash +void cachehash_set_evict_cb(cachehash *ch, cachehash_process_cb *cb); + +#ifdef __cplusplus +} +#endif + +#endif /* CACHEHASH_H */ diff --git a/lib/constraint.c b/lib/constraint.c new file mode 100644 index 0000000..503e17d --- /dev/null +++ b/lib/constraint.c @@ -0,0 +1,416 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../lib/constraint.h" +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +// +// Efficient address-space constraints (AH 7/2013) +// +// This module uses a tree-based representation to efficiently +// manipulate and query constraints on the address space to be +// scanned. It provides a value for every IP address, and these +// values are applied by setting them for network prefixes. Order +// matters: setting a value replaces any existing value for that +// prefix or subsets of it. We use this to implement network +// allowlisting and blocklisting. +// +// Think of setting values in this structure like painting +// subnets with different colors. We can paint subnets black to +// exclude them and white to allow them. Only the top color shows. +// This makes for potentially very powerful constraint specifications. +// +// Internally, this is implemented using a binary tree, where each +// node corresponds to a network prefix. (E.g., the root is +// 0.0.0.0/0, and its children, if present, are 0.0.0.0/1 and +// 128.0.0.0/1.) Each leaf of the tree stores the value that applies +// to every address within the leaf's portion of the prefix space. +// +// As an optimization, after all values are set, we look up the +// value or subtree for every /16 prefix and cache them as an array. +// This lets subsequent lookups bypass the bottom half of the tree. +// + +/* + * Constraint Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +typedef struct node { + struct node *l; + struct node *r; + value_t value; + uint64_t count; +} node_t; + +// As an optimization, we precompute lookups for every prefix of this +// length: +#define RADIX_LENGTH 20 + +struct _constraint { + node_t *root; // root node of the tree + uint32_t *radix; // array of prefixes (/RADIX_LENGTH) that are painted + // paint_value + size_t radix_len; // number of prefixes in radix array + int painted; // have we precomputed counts for each node? + value_t paint_value; // value for which we precomputed counts +}; + +// Tree operations respect the invariant that every node that isn't a +// leaf has exactly two children. +#define IS_LEAF(node) ((node)->l == NULL) + +// Allocate a new leaf with the given value +static node_t *_create_leaf(value_t value) +{ + node_t *node = xmalloc(sizeof(node_t)); + node->l = NULL; + node->r = NULL; + node->value = value; + return node; +} + +// Free the subtree rooted at node. +static void _destroy_subtree(node_t *node) +{ + if (node == NULL) + return; + _destroy_subtree(node->l); + _destroy_subtree(node->r); + free(node); +} + +// Convert from an internal node to a leaf. +static void _convert_to_leaf(node_t *node) +{ + assert(node); + assert(!IS_LEAF(node)); + _destroy_subtree(node->l); + _destroy_subtree(node->r); + node->l = NULL; + node->r = NULL; +} + +// Recursive function to set value for a given network prefix within +// the tree. (Note: prefix must be in host byte order.) +static void _set_recurse(node_t *node, uint32_t prefix, int len, value_t value) +{ + assert(node); + assert(0 <= len && len <= 256); + + if (len == 0) { + // We're at the end of the prefix; make this a leaf and set the + // value. + if (!IS_LEAF(node)) { + _convert_to_leaf(node); + } + node->value = value; + return; + } + + if (IS_LEAF(node)) { + // We're not at the end of the prefix, but we hit a leaf. + if (node->value == value) { + // A larger prefix has the same value, so we're done. + return; + } + // The larger prefix has a different value, so we need to + // convert it into an internal node and continue processing on + // one of the leaves. + node->l = _create_leaf(node->value); + node->r = _create_leaf(node->value); + } + + // We're not at the end of the prefix, and we're at an internal + // node. Recurse on the left or right subtree. + if (prefix & 0x80000000) { + _set_recurse(node->r, prefix << 1, len - 1, value); + } else { + _set_recurse(node->l, prefix << 1, len - 1, value); + } + + // At this point, we're an internal node, and the value is set + // by one of our children or its descendent. If both children are + // leaves with the same value, we can discard them and become a left. + if (IS_LEAF(node->r) && IS_LEAF(node->l) && + node->r->value == node->l->value) { + node->value = node->l->value; + _convert_to_leaf(node); + } +} + +// Set the value for a given network prefix, overwriting any existing +// values on that prefix or subsets of it. +// (Note: prefix must be in host byte order.) +void constraint_set(constraint_t *con, uint32_t prefix, int len, value_t value) +{ + assert(con); + _set_recurse(con->root, prefix, len, value); + con->painted = 0; +} + +// Return the value pertaining to an address, according to the tree +// starting at given root. (Note: address must be in host byte order.) +static int _lookup_ip(node_t *root, uint32_t address) +{ + assert(root); + node_t *node = root; + uint32_t mask = 0x80000000; + for (;;) { + if (IS_LEAF(node)) { + return node->value; + } + if (address & mask) { + node = node->r; + } else { + node = node->l; + } + mask >>= 1; + } +} + +// Return the value pertaining to an address. +// (Note: address must be in host byte order.) +value_t constraint_lookup_ip(constraint_t *con, uint32_t address) +{ + assert(con); + return _lookup_ip(con->root, address); +} + +// Return the nth painted IP address. +static int _lookup_index(node_t *root, uint64_t n) +{ + assert(root); + node_t *node = root; + uint32_t ip = 0; + uint32_t mask = 0x80000000; + for (;;) { + if (IS_LEAF(node)) { + return ip | n; + } + if (n < node->l->count) { + node = node->l; + } else { + n -= node->l->count; + node = node->r; + ip |= mask; + } + mask >>= 1; + } +} + +// For a given value, return the IP address with zero-based index n. +// (i.e., if there are three addresses with value 0xFF, looking up index 1 +// will return the second one). +// Note that the tree must have been previously painted with this value. +uint32_t constraint_lookup_index(constraint_t *con, uint64_t index, + value_t value) +{ + assert(con); + if (!con->painted || con->paint_value != value) { + constraint_paint_value(con, value); + } + + uint64_t radix_idx = index / (1 << (32 - RADIX_LENGTH)); + if (radix_idx < con->radix_len) { + // Radix lookup + uint32_t radix_offset = + index % (1 << (32 - RADIX_LENGTH)); // TODO: bitwise maths + return con->radix[radix_idx] | radix_offset; + } + + // Otherwise, do the "slow" lookup in tree. + // Note that tree counts do NOT include things in the radix, + // so we subtract these off here. + index -= con->radix_len * (1 << (32 - RADIX_LENGTH)); + assert(index < con->root->count); + return _lookup_index(con->root, index); +} + +// Implement count_ips by recursing on halves of the tree. Size represents +// the number of addresses in a prefix at the current level of the tree. +// If paint is specified, each node will have its count set to the number of +// leaves under it set to value. +// If exclude_radix is specified, the number of addresses will exclude prefixes +// that are a /RADIX_LENGTH or larger +static uint64_t _count_ips_recurse(node_t *node, value_t value, uint64_t size, + int paint, int exclude_radix) +{ + assert(node); + uint64_t n; + if (IS_LEAF(node)) { + if (node->value == value) { + n = size; + // Exclude prefixes already included in the radix + if (exclude_radix && + size >= (1 << (32 - RADIX_LENGTH))) { + n = 0; + } + } else { + n = 0; + } + } else { + n = _count_ips_recurse(node->l, value, size >> 1, paint, + exclude_radix) + + _count_ips_recurse(node->r, value, size >> 1, paint, + exclude_radix); + } + if (paint) { + node->count = n; + } + return n; +} + +// Return a node that determines the values for the addresses with +// the given prefix. This is either the internal node that +// corresponds to the end of the prefix or a leaf node that +// encompasses the prefix. (Note: prefix must be in host byte order.) +static node_t *_lookup_node(node_t *root, uint32_t prefix, int len) +{ + assert(root); + assert(0 <= len && len <= 32); + + node_t *node = root; + uint32_t mask = 0x80000000; + int i; + + for (i = 0; i < len; i++) { + if (IS_LEAF(node)) { + return node; + } + if (prefix & mask) { + node = node->r; + } else { + node = node->l; + } + mask >>= 1; + } + return node; +} + +// For each node, precompute the count of leaves beneath it set to value. +// Note that the tree can be painted for only one value at a time. +void constraint_paint_value(constraint_t *con, value_t value) +{ + assert(con); + log_debug("constraint", "Painting value %lu", value); + + // Paint everything except what we will put in radix + _count_ips_recurse(con->root, value, (uint64_t)1 << 32, 1, 1); + + // Fill in the radix array with a list of addresses + uint32_t i; + con->radix_len = 0; + for (i = 0; i < (1 << RADIX_LENGTH); i++) { + uint32_t prefix = i << (32 - RADIX_LENGTH); + node_t *node = _lookup_node(con->root, prefix, RADIX_LENGTH); + if (IS_LEAF(node) && node->value == value) { + // Add this prefix to the radix + con->radix[con->radix_len++] = prefix; + } + } + log_debug("constraint", "%lu IPs in radix array, %lu IPs in tree", + con->radix_len * (1 << (32 - RADIX_LENGTH)), + con->root->count); + con->painted = 1; + con->paint_value = value; +} + +// Return the number of addresses that have a given value. +uint64_t constraint_count_ips(constraint_t *con, value_t value) +{ + assert(con); + if (con->painted && con->paint_value == value) { + return con->root->count + + con->radix_len * (1 << (32 - RADIX_LENGTH)); + } else { + return _count_ips_recurse(con->root, value, (uint64_t)1 << 32, + 0, 0); + } +} + +// Initialize the tree. +// All addresses will initially have the given value. +constraint_t *constraint_init(value_t value) +{ + constraint_t *con = xmalloc(sizeof(constraint_t)); + con->root = _create_leaf(value); + con->radix = xcalloc(sizeof(uint32_t), 1 << RADIX_LENGTH); + con->painted = 0; + return con; +} + +// Deinitialize and free the tree. +void constraint_free(constraint_t *con) +{ + assert(con); + log_debug("constraint", "Cleaning up"); + _destroy_subtree(con->root); + free(con->radix); + free(con); +} + +/* +int main(void) +{ + log_init(stderr, LOG_DEBUG); + + constraint_t *con = constraint_init(0); + constraint_set(con, ntohl(inet_addr("128.128.0.0")), 1, 22); + constraint_set(con, ntohl(inet_addr("128.128.0.0")), 1, 1); + constraint_set(con, ntohl(inet_addr("128.0.0.0")), 1, 1); + constraint_set(con, ntohl(inet_addr("10.0.0.0")), 24, 1); + constraint_set(con, ntohl(inet_addr("10.0.0.0")), 24, 0); + constraint_set(con, ntohl(inet_addr("10.11.12.0")), 24, 1); + constraint_set(con, ntohl(inet_addr("141.212.0.0")), 16, 0); + + for (int x=1; x < 2; x++) { + if (x == 1) { + constraint_optimize(con); + } + + printf("count(0)=%ld\n", constraint_count_ips(con, 0)); + printf("count(1)=%ld\n", constraint_count_ips(con, 1)); + printf("%d\n", +constraint_lookup_ip(con,ntohl(inet_addr("10.11.12.0")))); + assert(constraint_count_ips(con, 0) + constraint_count_ips(con, +1) == (uint64_t)1 << 32); + + uint32_t i=0, count=0; + do { + if (constraint_lookup_ip(con, i)) + count++; + } while (++i != 0); + printf("derived count(1)=%u\n", count); + } + + constraint_free(con); +} +*/ diff --git a/lib/constraint.h b/lib/constraint.h new file mode 100644 index 0000000..57e9b88 --- /dev/null +++ b/lib/constraint.h @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONSTRAINT_H +#define CONSTRAINT_H + +#include + +typedef struct _constraint constraint_t; +typedef uint32_t value_t; + +constraint_t *constraint_init(value_t value); +void constraint_free(constraint_t *con); +void constraint_set(constraint_t *con, uint32_t prefix, int len, value_t value); +value_t constraint_lookup_ip(constraint_t *con, uint32_t address); +uint64_t constraint_count_ips(constraint_t *con, value_t value); +uint32_t constraint_lookup_index(constraint_t *con, uint64_t index, + value_t value); +void constraint_paint_value(constraint_t *con, value_t value); + +#endif //_CONSTRAINT_H diff --git a/lib/csv.c b/lib/csv.c new file mode 100644 index 0000000..190f6f3 --- /dev/null +++ b/lib/csv.c @@ -0,0 +1,52 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "csv.h" + +int csv_find_index(char *header, const char **names, size_t names_len) +{ + char *split = header; + for (int idx = 0; split != NULL; ++idx) { + char *front = (idx == 0) ? split : split + 1; + for (size_t i = 0; i < names_len; ++i) { + if (strncmp(front, names[i], strlen(names[i])) == 0) { + return idx; + } + } + split = strchr(front, ','); + } + return -1; +} + +char *csv_get_index(char *row, size_t idx) +{ + char *split = row; + for (size_t i = 0; i < idx; ++i) { + split = strchr(split + 1, ','); + if (split == NULL) { + return NULL; + } + } + char *entry; + char *start = (idx == 0) ? split : split + 1; + char *end = strchr(start, ','); + if (end != NULL) { + entry = strndup(start, end - start); + } else { + entry = strdup(start); + } + return entry; +} diff --git a/lib/csv.h b/lib/csv.h new file mode 100644 index 0000000..9229cdc --- /dev/null +++ b/lib/csv.h @@ -0,0 +1,25 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_CSV_H +#define ZMAP_CSV_H + +#include + +int csv_find_index(char *header, const char **names, size_t names_len); +char *csv_get_index(char *row, size_t idx); + +#endif diff --git a/lib/includes.h b/lib/includes.h new file mode 100644 index 0000000..f2c9aa8 --- /dev/null +++ b/lib/includes.h @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FAVOR_BSD +#define __FAVOR_BSD 2 +#endif +#ifndef __USE_BSD +#define __USE_BSD +#endif + +#include +#include +#include +#include +#include +#include +#include + +#if defined(__NetBSD__) +#define ICMP_UNREACH_PRECEDENCE_CUTOFF ICMP_UNREACH_PREC_CUTOFF +#include +#else +#include +#endif + +#include +#include +#include // NOTE: net/if.h MUST be included BEFORE ifaddrs.h +#include + +#define MAC_ADDR_LEN ETHER_ADDR_LEN +#define UNUSED __attribute__((unused)) diff --git a/lib/lockfd.c b/lib/lockfd.c new file mode 100644 index 0000000..093e819 --- /dev/null +++ b/lib/lockfd.c @@ -0,0 +1,57 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "xalloc.h" + +static pthread_mutex_t **mutexes = NULL; + +static pthread_mutex_t *get_mutex(int fd) +{ + assert(fd < 3 && "todo: implement generically"); + if (!mutexes) { + mutexes = xmalloc(3 * sizeof(char *)); + assert(mutexes); + } + if (!mutexes[fd]) { + mutexes[fd] = xmalloc(sizeof(pthread_mutex_t)); + assert(mutexes[fd]); + pthread_mutex_init(mutexes[fd], NULL); + assert(mutexes[fd]); + } + return mutexes[fd]; +} + +int lock_fd(int fd) { return pthread_mutex_lock(get_mutex(fd)); } + +int unlock_fd(int fd) { return pthread_mutex_unlock(get_mutex(fd)); } + +int lock_file(FILE *f) +{ + assert(f); + return lock_fd(fileno(f)); +} + +int unlock_file(FILE *f) +{ + assert(f); + return unlock_fd(fileno(f)); +} diff --git a/lib/lockfd.h b/lib/lockfd.h new file mode 100644 index 0000000..625534a --- /dev/null +++ b/lib/lockfd.h @@ -0,0 +1,22 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +int lock_fd(int fd); +int unlock_fd(int fd); +int lock_file(FILE *f); +int unlock_file(FILE *f); diff --git a/lib/logger.c b/lib/logger.c new file mode 100644 index 0000000..dc8f047 --- /dev/null +++ b/lib/logger.c @@ -0,0 +1,271 @@ +/* + * Logger Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "xalloc.h" +#include "lockfd.h" + +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static enum LogLevel log_output_level = ZLOG_INFO; + +static FILE *log_output_stream = NULL; +static int color = 0; + +static int log_to_syslog = 0; + +static const char *log_level_name[] = {"FATAL", "ERROR", "WARN", + "INFO", "DEBUG", "TRACE"}; + +#define RED "\x1b[31m" +#define GREEN "\x1b[32m" +#define YELLOW "\x1b[33m" +#define BLUE "\x1b[34m" +#define MAGENTA "\x1b[35m" +#define CYAN "\x1b[36m" +#define RESET "\033[0m" + +#define COLOR(x) \ + do { \ + if (color) \ + fprintf(log_output_stream, "%s", x); \ + } while (0) + +static const char *color_for_level(enum LogLevel level) +{ + switch (level) { + case ZLOG_FATAL: + return RED; + case ZLOG_ERROR: + return MAGENTA; + case ZLOG_WARN: + return YELLOW; + case ZLOG_INFO: + return GREEN; + case ZLOG_DEBUG: + return BLUE; + case ZLOG_TRACE: + return RESET; + default: + return RESET; + } +} + +static int LogLogVA(enum LogLevel level, const char *loggerName, + const char *logMessage, va_list args) +{ + if (level <= log_output_level) { + if (!log_output_stream) { + log_output_stream = stderr; + } + // if logging to a shared output channel, then use a global + // lock across ZMap. Otherwise, if we're logging to a file, + // only lockin with the module, in order to avoid having + // corrupt log entries. + if (log_output_stream == stdout || + log_output_stream == stderr) { + lock_file(log_output_stream); + } else { + pthread_mutex_lock(&mutex); + } + if (color) { + COLOR(color_for_level(level)); + } + + const char *levelName = log_level_name[level]; + struct timeval now; + char timestamp[256]; + gettimeofday(&now, NULL); + time_t sec = now.tv_sec; + struct tm *ptm = localtime(&sec); + strftime(timestamp, 20, "%b %d %H:%M:%S", ptm); + fprintf(log_output_stream, "%s.%03ld [%s] ", timestamp, + (long)now.tv_usec / 1000, levelName); + if (loggerName) { + fprintf(log_output_stream, "%s: ", loggerName); + } + if (logMessage) { + vfprintf(log_output_stream, logMessage, args); + } + if (loggerName || logMessage) { + fputs("\n", log_output_stream); + } + if (color) { + COLOR(RESET); + } + fflush(log_output_stream); + if (log_output_stream == stdout || + log_output_stream == stderr) { + unlock_file(log_output_stream); + } else { + pthread_mutex_unlock(&mutex); + } + } + return EXIT_SUCCESS; +} + +int log_fatal(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + LogLogVA(ZLOG_FATAL, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_CRIT), message, va); + va_end(va); + } + + exit(EXIT_FAILURE); +} + +int log_error(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_ERROR, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_ERR), message, va); + va_end(va); + } + + return ret; +} + +int log_warn(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_WARN, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_WARNING), message, va); + va_end(va); + } + + return ret; +} + +int log_info(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_INFO, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_INFO), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} + +int log_debug(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_DEBUG, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_DEBUG), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} + +#ifdef DEBUG +extern int log_trace(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_TRACE, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_DEBUG), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} +#endif + +int log_init(FILE *stream, enum LogLevel level, int syslog_enabled, + const char *appname) +{ + log_output_stream = stream; + log_output_level = level; + if (syslog_enabled) { + log_to_syslog = 1; + openlog(appname, 0, LOG_USER); // no options + } + if (isatty(fileno(log_output_stream))) { + color = 1; + } + return 0; +} + +void check_and_log_file_error(FILE *file, const char *name) +{ + if (ferror(file)) { + log_fatal(name, "unable to write to file"); + } +} + +size_t dstrftime(char *buf, size_t maxsize, const char *format, double tm) +{ + struct timeval tv; + double tm_floor; + tm_floor = floor(tm); + tv.tv_sec = (long)tm_floor; + tv.tv_usec = (long)(tm - floor(tm)) * 1000000; + return strftime(buf, maxsize, format, localtime((const time_t *)&tv)); +} diff --git a/lib/logger.h b/lib/logger.h new file mode 100644 index 0000000..f4daf47 --- /dev/null +++ b/lib/logger.h @@ -0,0 +1,67 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#ifndef LOGGER_H +#define LOGGER_H + +#ifdef __cplusplus +extern "C" { +#endif + +// do not collide with constants defined in syslog.h +enum LogLevel { + ZLOG_FATAL, + ZLOG_ERROR, + ZLOG_WARN, + ZLOG_INFO, + ZLOG_DEBUG, + ZLOG_TRACE, + ZNUM_LOGLEVELS +}; + +int log_fatal(const char *loggerName, const char *logMessage, ...) + __attribute__((noreturn)); + +int log_error(const char *loggerName, const char *logMessage, ...); + +int log_warn(const char *loggerName, const char *logMessage, ...); + +int log_info(const char *loggerName, const char *logMessage, ...); + +int log_debug(const char *loggerName, const char *logMessage, ...); + +#ifdef DEBUG +int log_trace(const char *loggerName, const char *logMessage, ...); +#else +#define log_trace(...) ; +#endif + +int log_init(FILE *stream, enum LogLevel level, int syslog_enabled, + const char *syslog_app); + +void check_and_log_file_error(FILE *file, const char *name); + +size_t dstrftime(char *, size_t, const char *, double); + +#ifdef __cplusplus +} +#endif + +#endif // _LOGGER_H diff --git a/lib/pbm.c b/lib/pbm.c new file mode 100644 index 0000000..212c380 --- /dev/null +++ b/lib/pbm.c @@ -0,0 +1,111 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "logger.h" +#include "xalloc.h" + +#define NUM_VALUES 0xFFFFFFFF +#define PAGE_SIZE_IN_BITS 0x10000 +#define PAGE_SIZE_IN_BYTES (PAGE_SIZE_IN_BITS / 8) +#define NUM_PAGES 0x10000 +#define PAGE_MASK 0xFFFF + +uint8_t **pbm_init(void) +{ + uint8_t **retv = xcalloc(NUM_PAGES, sizeof(void *)); + return retv; +} + +uint8_t *bm_init(void) +{ + uint8_t *bm = xmalloc(PAGE_SIZE_IN_BYTES); + memset(bm, 0, PAGE_SIZE_IN_BYTES); + return bm; +} + +int bm_check(uint8_t *bm, uint16_t v) +{ + uint16_t page_idx = (v >> 3); + uint8_t bit_idx = (uint8_t)(v & 0x07); + return bm[page_idx] & (1 << bit_idx); +} + +void bm_set(uint8_t *bm, uint16_t v) +{ + uint16_t page_idx = (v >> 3); + uint8_t bit_idx = (uint8_t)(v & 0x07); + bm[page_idx] |= (1 << bit_idx); +} + +int pbm_check(uint8_t **b, uint32_t v) +{ + uint32_t top = v >> 16; + uint32_t bottom = v & PAGE_MASK; + return b[top] && bm_check(b[top], bottom); +} + +void pbm_set(uint8_t **b, uint32_t v) +{ + uint16_t top = (uint16_t)(v >> 16); + uint16_t bottom = (uint16_t)(v & PAGE_MASK); + if (!b[top]) { + b[top] = bm_init(); + } + bm_set(b[top], bottom); +} + +uint32_t pbm_load_from_file(uint8_t **b, char *file) +{ + if (!b) { + log_fatal("pbm", "load_from_file called with NULL PBM"); + } + if (!file) { + log_fatal("pbm", "load_from_file called with NULL filename"); + } + FILE *fp = fopen(file, "r"); + if (fp == NULL) { + log_fatal("pbm", "unable to open file: %s: %s", file, + strerror(errno)); + } + char line[1000]; + uint32_t count = 0; + while (fgets(line, sizeof(line), fp)) { + char *comment = strchr(line, '#'); + if (comment) { + *comment = '\0'; + } + struct in_addr addr; + if (inet_aton(line, &addr) != 1) { + log_fatal("pbm", "unable to parse IP address: %s", + line); + } + pbm_set(b, addr.s_addr); + ++count; + } + fclose(fp); + return count; +} diff --git a/lib/pbm.h b/lib/pbm.h new file mode 100644 index 0000000..59179a5 --- /dev/null +++ b/lib/pbm.h @@ -0,0 +1,31 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_PBM_H +#define ZMAP_PBM_H + +#include + +uint8_t *bm_init(void); +int bm_check(uint8_t *bm, uint16_t v); +void bm_set(uint8_t *bm, uint16_t v); + +uint8_t **pbm_init(void); +int pbm_check(uint8_t **b, uint32_t v); +void pbm_set(uint8_t **b, uint32_t v); +uint32_t pbm_load_from_file(uint8_t **b, char *file); + +#endif /* ZMAP_PBM_H */ diff --git a/lib/queue.c b/lib/queue.c new file mode 100644 index 0000000..874e286 --- /dev/null +++ b/lib/queue.c @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "queue.h" + +#include "xalloc.h" + +#include + +zqueue_t *queue_init(void) +{ + zqueue_t *p = xmalloc(sizeof(zqueue_t)); + p->front = NULL; + p->back = NULL; + p->size = 0; + + pthread_mutex_init(&p->lock, NULL); + pthread_cond_init(&p->empty, NULL); + return p; +} + +int is_empty(zqueue_t *queue) { return queue->size == 0; } + +void push_back(char *data, zqueue_t *queue) +{ + znode_t *new_node = xmalloc(sizeof(znode_t)); + new_node->prev = NULL; + new_node->next = NULL; + new_node->data = strdup(data); + + pthread_mutex_lock(&queue->lock); + if (is_empty(queue)) { + queue->front = new_node; + queue->back = new_node; + } else { + queue->back->next = new_node; + new_node->prev = queue->back; + queue->back = new_node; + } + queue->size++; + pthread_cond_signal(&queue->empty); + pthread_mutex_unlock(&queue->lock); +} + +znode_t *pop_front(zqueue_t *queue) +{ + pthread_mutex_lock(&queue->lock); + + while (is_empty(queue)) { + pthread_cond_wait(&queue->empty, &queue->lock); + } + znode_t *temp = pop_front_unsafe(queue); + pthread_mutex_unlock(&queue->lock); + return temp; +} + +znode_t *pop_front_unsafe(zqueue_t *queue) +{ + znode_t *temp = queue->front; + queue->front = temp->next; + if (queue->front != NULL) { + queue->front->prev = NULL; + } + queue->size--; + return temp; +} + +znode_t *get_front(zqueue_t *queue) +{ + pthread_mutex_lock(&queue->lock); + + while (is_empty(queue)) { + pthread_cond_wait(&queue->empty, &queue->lock); + } + + znode_t *temp = xmalloc(sizeof(znode_t)); + temp = queue->front; + pthread_mutex_unlock(&queue->lock); + return temp; +} + +znode_t *get_back(zqueue_t *queue) +{ + pthread_mutex_lock(&queue->lock); + + while (is_empty(queue)) { + pthread_cond_wait(&queue->empty, &queue->lock); + } + + znode_t *temp = xmalloc(sizeof(znode_t)); + temp = queue->back; + pthread_mutex_unlock(&queue->lock); + return temp; +} + +size_t get_size(zqueue_t *queue) { return queue->size; } diff --git a/lib/queue.h b/lib/queue.h new file mode 100644 index 0000000..7a375c9 --- /dev/null +++ b/lib/queue.h @@ -0,0 +1,49 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_QUEUE_H +#define ZMAP_QUEUE_H + +#include +#include +#include +#include + +typedef struct zqueue_node { + char *data; + struct zqueue_node *prev; + struct zqueue_node *next; +} znode_t; + +typedef struct zqueue { + struct zqueue_node *front; + struct zqueue_node *back; + size_t size; + // Threading utilities + pthread_mutex_t lock; + pthread_cond_t empty; +} zqueue_t; + +zqueue_t *queue_init(void); +int is_empty(zqueue_t *queue); +void push_back(char *data, zqueue_t *queue); +znode_t *pop_front(zqueue_t *queue); +znode_t *pop_front_unsafe(zqueue_t *queue); +znode_t *get_front(zqueue_t *queue); +znode_t *get_back(zqueue_t *queue); +size_t get_size(zqueue_t *queue); + +#endif /* ZMAP_QUEUE_H */ diff --git a/lib/random.c b/lib/random.c new file mode 100644 index 0000000..e3ac8f8 --- /dev/null +++ b/lib/random.c @@ -0,0 +1,32 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "random.h" +#include +#include +#include +#include +#include +#include "logger.h" + +#define RANDSRC "/dev/urandom" + +int random_bytes(void *dst, size_t n) +{ + FILE *f = fopen(RANDSRC, "rb"); + if (!f) { + log_fatal("random", "unable to read /dev/urandom: %s", + strerror(errno)); + } + size_t r = fread(dst, n, 1, f); + fclose(f); + if (r < 1) { + return 0; + } + return 1; +} diff --git a/lib/random.h b/lib/random.h new file mode 100644 index 0000000..1151efe --- /dev/null +++ b/lib/random.h @@ -0,0 +1,25 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#ifndef RANDOM_H +#define RANDOM_H + +int random_bytes(void *dst, size_t n); + +#endif diff --git a/lib/rijndael-alg-fst.c b/lib/rijndael-alg-fst.c new file mode 100644 index 0000000..4a36327 --- /dev/null +++ b/lib/rijndael-alg-fst.c @@ -0,0 +1,1284 @@ +/** + * rijndael-alg-fst.c + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen + * @author Antoon Bosselaers + * @author Paulo Barreto + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include + +#include "rijndael-alg-fst.h" + +/* +Te0[x] = S [x].[02, 01, 01, 03]; +Te1[x] = S [x].[03, 02, 01, 01]; +Te2[x] = S [x].[01, 03, 02, 01]; +Te3[x] = S [x].[01, 01, 03, 02]; +Te4[x] = S [x].[01, 01, 01, 01]; + +Td0[x] = Si[x].[0e, 09, 0d, 0b]; +Td1[x] = Si[x].[0b, 0e, 09, 0d]; +Td2[x] = Si[x].[0d, 0b, 0e, 09]; +Td3[x] = Si[x].[09, 0d, 0b, 0e]; +Td4[x] = Si[x].[01, 01, 01, 01]; +*/ + +static const u32 Te0[256] = { + 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, 0xfff2f20dU, + 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, 0x60303050U, 0x02010103U, + 0xce6767a9U, 0x562b2b7dU, 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, + 0xec76769aU, 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, + 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, 0x41adadecU, + 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, 0x239c9cbfU, 0x53a4a4f7U, + 0xe4727296U, 0x9bc0c05bU, 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, + 0x4c26266aU, 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, + 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, 0xe2717193U, + 0xabd8d873U, 0x62313153U, 0x2a15153fU, 0x0804040cU, 0x95c7c752U, + 0x46232365U, 0x9dc3c35eU, 0x30181828U, 0x379696a1U, 0x0a05050fU, + 0x2f9a9ab5U, 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, + 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, 0x1209091bU, + 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, 0x361b1b2dU, 0xdc6e6eb2U, + 0xb45a5aeeU, 0x5ba0a0fbU, 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, + 0x7db3b3ceU, 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, + 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, 0x40202060U, + 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, 0xd46a6abeU, 0x8dcbcb46U, + 0x67bebed9U, 0x7239394bU, 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, + 0x85cfcf4aU, 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, + 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, 0x8a4545cfU, + 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, 0xa05050f0U, 0x783c3c44U, + 0x259f9fbaU, 0x4ba8a8e3U, 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, + 0x058f8f8aU, 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, + 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, 0x20101030U, + 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, 0x81cdcd4cU, 0x180c0c14U, + 0x26131335U, 0xc3ecec2fU, 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, + 0x2e171739U, 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, + 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, 0xc06060a0U, + 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, 0x44222266U, 0x542a2a7eU, + 0x3b9090abU, 0x0b888883U, 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, + 0x2814143cU, 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, + 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, 0x924949dbU, + 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, 0x9fc2c25dU, 0xbdd3d36eU, + 0x43acacefU, 0xc46262a6U, 0x399191a8U, 0x319595a4U, 0xd3e4e437U, + 0xf279798bU, 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, + 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, 0xd86c6cb4U, + 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, 0xca6565afU, 0xf47a7a8eU, + 0x47aeaee9U, 0x10080818U, 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, + 0x5c2e2e72U, 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, + 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, 0x964b4bddU, + 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, 0xe0707090U, 0x7c3e3e42U, + 0x71b5b5c4U, 0xcc6666aaU, 0x904848d8U, 0x06030305U, 0xf7f6f601U, + 0x1c0e0e12U, 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, + 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, 0xd9e1e138U, + 0xebf8f813U, 0x2b9898b3U, 0x22111133U, 0xd26969bbU, 0xa9d9d970U, + 0x078e8e89U, 0x339494a7U, 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, + 0xc9e9e920U, 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, + 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, 0x65bfbfdaU, + 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, 0x824141c3U, 0x299999b0U, + 0x5a2d2d77U, 0x1e0f0f11U, 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, + 0x2c16163aU, +}; +static const u32 Te1[256] = { + 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, 0x0dfff2f2U, + 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, 0x50603030U, 0x03020101U, + 0xa9ce6767U, 0x7d562b2bU, 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, + 0x9aec7676U, 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, + 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, 0xec41adadU, + 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, 0xbf239c9cU, 0xf753a4a4U, + 0x96e47272U, 0x5b9bc0c0U, 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, + 0x6a4c2626U, 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, + 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, 0x93e27171U, + 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, 0x0c080404U, 0x5295c7c7U, + 0x65462323U, 0x5e9dc3c3U, 0x28301818U, 0xa1379696U, 0x0f0a0505U, + 0xb52f9a9aU, 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, + 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, 0x1b120909U, + 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, 0x2d361b1bU, 0xb2dc6e6eU, + 0xeeb45a5aU, 0xfb5ba0a0U, 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, + 0xce7db3b3U, 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, + 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, 0x60402020U, + 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, 0xbed46a6aU, 0x468dcbcbU, + 0xd967bebeU, 0x4b723939U, 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, + 0x4a85cfcfU, 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, + 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, 0xcf8a4545U, + 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, 0xf0a05050U, 0x44783c3cU, + 0xba259f9fU, 0xe34ba8a8U, 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, + 0x8a058f8fU, 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, + 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, 0x30201010U, + 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, 0x4c81cdcdU, 0x14180c0cU, + 0x35261313U, 0x2fc3ececU, 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, + 0x392e1717U, 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, + 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, 0xa0c06060U, + 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, 0x66442222U, 0x7e542a2aU, + 0xab3b9090U, 0x830b8888U, 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, + 0x3c281414U, 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, + 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, 0xdb924949U, + 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, 0x5d9fc2c2U, 0x6ebdd3d3U, + 0xef43acacU, 0xa6c46262U, 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, + 0x8bf27979U, 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, + 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, 0xb4d86c6cU, + 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, 0xafca6565U, 0x8ef47a7aU, + 0xe947aeaeU, 0x18100808U, 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, + 0x725c2e2eU, 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, + 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, 0xdd964b4bU, + 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, 0x90e07070U, 0x427c3e3eU, + 0xc471b5b5U, 0xaacc6666U, 0xd8904848U, 0x05060303U, 0x01f7f6f6U, + 0x121c0e0eU, 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, + 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, 0x38d9e1e1U, + 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, 0xbbd26969U, 0x70a9d9d9U, + 0x89078e8eU, 0xa7339494U, 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, + 0x20c9e9e9U, 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, + 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, 0xda65bfbfU, + 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, 0xc3824141U, 0xb0299999U, + 0x775a2d2dU, 0x111e0f0fU, 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, + 0x3a2c1616U, +}; +static const u32 Te2[256] = { + 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, 0xf20dfff2U, + 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, 0x30506030U, 0x01030201U, + 0x67a9ce67U, 0x2b7d562bU, 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, + 0x769aec76U, 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, + 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, 0xadec41adU, + 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, 0x9cbf239cU, 0xa4f753a4U, + 0x7296e472U, 0xc05b9bc0U, 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, + 0x266a4c26U, 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, + 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, 0x7193e271U, + 0xd873abd8U, 0x31536231U, 0x153f2a15U, 0x040c0804U, 0xc75295c7U, + 0x23654623U, 0xc35e9dc3U, 0x18283018U, 0x96a13796U, 0x050f0a05U, + 0x9ab52f9aU, 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, + 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, 0x091b1209U, + 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, 0x1b2d361bU, 0x6eb2dc6eU, + 0x5aeeb45aU, 0xa0fb5ba0U, 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, + 0xb3ce7db3U, 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, + 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, 0x20604020U, + 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, 0x6abed46aU, 0xcb468dcbU, + 0xbed967beU, 0x394b7239U, 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, + 0xcf4a85cfU, 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, + 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, 0x45cf8a45U, + 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, 0x50f0a050U, 0x3c44783cU, + 0x9fba259fU, 0xa8e34ba8U, 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, + 0x8f8a058fU, 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, + 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, 0x10302010U, + 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, 0xcd4c81cdU, 0x0c14180cU, + 0x13352613U, 0xec2fc3ecU, 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, + 0x17392e17U, 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, + 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, 0x60a0c060U, + 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, 0x22664422U, 0x2a7e542aU, + 0x90ab3b90U, 0x88830b88U, 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, + 0x143c2814U, 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, + 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, 0x49db9249U, + 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, 0xc25d9fc2U, 0xd36ebdd3U, + 0xacef43acU, 0x62a6c462U, 0x91a83991U, 0x95a43195U, 0xe437d3e4U, + 0x798bf279U, 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, + 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, 0x6cb4d86cU, + 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, 0x65afca65U, 0x7a8ef47aU, + 0xaee947aeU, 0x08181008U, 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, + 0x2e725c2eU, 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, + 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, 0x4bdd964bU, + 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, 0x7090e070U, 0x3e427c3eU, + 0xb5c471b5U, 0x66aacc66U, 0x48d89048U, 0x03050603U, 0xf601f7f6U, + 0x0e121c0eU, 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, + 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, 0xe138d9e1U, + 0xf813ebf8U, 0x98b32b98U, 0x11332211U, 0x69bbd269U, 0xd970a9d9U, + 0x8e89078eU, 0x94a73394U, 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, + 0xe920c9e9U, 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, + 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, 0xbfda65bfU, + 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, 0x41c38241U, 0x99b02999U, + 0x2d775a2dU, 0x0f111e0fU, 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, + 0x163a2c16U, +}; +static const u32 Te3[256] = { + + 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, 0xf2f20dffU, + 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, 0x30305060U, 0x01010302U, + 0x6767a9ceU, 0x2b2b7d56U, 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, + 0x76769aecU, 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, + 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, 0xadadec41U, + 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, 0x9c9cbf23U, 0xa4a4f753U, + 0x727296e4U, 0xc0c05b9bU, 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, + 0x26266a4cU, 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, + 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, 0x717193e2U, + 0xd8d873abU, 0x31315362U, 0x15153f2aU, 0x04040c08U, 0xc7c75295U, + 0x23236546U, 0xc3c35e9dU, 0x18182830U, 0x9696a137U, 0x05050f0aU, + 0x9a9ab52fU, 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, + 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, 0x09091b12U, + 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, 0x1b1b2d36U, 0x6e6eb2dcU, + 0x5a5aeeb4U, 0xa0a0fb5bU, 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, + 0xb3b3ce7dU, 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, + 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, 0x20206040U, + 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, 0x6a6abed4U, 0xcbcb468dU, + 0xbebed967U, 0x39394b72U, 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, + 0xcfcf4a85U, 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, + 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, 0x4545cf8aU, + 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, 0x5050f0a0U, 0x3c3c4478U, + 0x9f9fba25U, 0xa8a8e34bU, 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, + 0x8f8f8a05U, 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, + 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, 0x10103020U, + 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, 0xcdcd4c81U, 0x0c0c1418U, + 0x13133526U, 0xecec2fc3U, 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, + 0x1717392eU, 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, + 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, 0x6060a0c0U, + 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, 0x22226644U, 0x2a2a7e54U, + 0x9090ab3bU, 0x8888830bU, 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, + 0x14143c28U, 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, + 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, 0x4949db92U, + 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, 0xc2c25d9fU, 0xd3d36ebdU, + 0xacacef43U, 0x6262a6c4U, 0x9191a839U, 0x9595a431U, 0xe4e437d3U, + 0x79798bf2U, 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, + 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, 0x6c6cb4d8U, + 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, 0x6565afcaU, 0x7a7a8ef4U, + 0xaeaee947U, 0x08081810U, 0xbabad56fU, 0x787888f0U, 0x25256f4aU, + 0x2e2e725cU, 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, + 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, 0x4b4bdd96U, + 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, 0x707090e0U, 0x3e3e427cU, + 0xb5b5c471U, 0x6666aaccU, 0x4848d890U, 0x03030506U, 0xf6f601f7U, + 0x0e0e121cU, 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, + 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, 0xe1e138d9U, + 0xf8f813ebU, 0x9898b32bU, 0x11113322U, 0x6969bbd2U, 0xd9d970a9U, + 0x8e8e8907U, 0x9494a733U, 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, + 0xe9e920c9U, 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, + 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, 0xbfbfda65U, + 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, 0x4141c382U, 0x9999b029U, + 0x2d2d775aU, 0x0f0f111eU, 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, + 0x16163a2cU, +}; +static const u32 Te4[256] = { + 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, 0xf2f2f2f2U, + 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, 0x30303030U, 0x01010101U, + 0x67676767U, 0x2b2b2b2bU, 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, + 0x76767676U, 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, + 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, 0xadadadadU, + 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, 0x9c9c9c9cU, 0xa4a4a4a4U, + 0x72727272U, 0xc0c0c0c0U, 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, + 0x26262626U, 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, + 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, 0x71717171U, + 0xd8d8d8d8U, 0x31313131U, 0x15151515U, 0x04040404U, 0xc7c7c7c7U, + 0x23232323U, 0xc3c3c3c3U, 0x18181818U, 0x96969696U, 0x05050505U, + 0x9a9a9a9aU, 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, + 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, 0x09090909U, + 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, 0x1b1b1b1bU, 0x6e6e6e6eU, + 0x5a5a5a5aU, 0xa0a0a0a0U, 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, + 0xb3b3b3b3U, 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, + 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, 0x20202020U, + 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, 0x6a6a6a6aU, 0xcbcbcbcbU, + 0xbebebebeU, 0x39393939U, 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, + 0xcfcfcfcfU, 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, + 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, 0x45454545U, + 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, 0x50505050U, 0x3c3c3c3cU, + 0x9f9f9f9fU, 0xa8a8a8a8U, 0x51515151U, 0xa3a3a3a3U, 0x40404040U, + 0x8f8f8f8fU, 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, + 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, 0x10101010U, + 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, 0xcdcdcdcdU, 0x0c0c0c0cU, + 0x13131313U, 0xececececU, 0x5f5f5f5fU, 0x97979797U, 0x44444444U, + 0x17171717U, 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, + 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, 0x60606060U, + 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, 0x22222222U, 0x2a2a2a2aU, + 0x90909090U, 0x88888888U, 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, + 0x14141414U, 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, + 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, 0x49494949U, + 0x06060606U, 0x24242424U, 0x5c5c5c5cU, 0xc2c2c2c2U, 0xd3d3d3d3U, + 0xacacacacU, 0x62626262U, 0x91919191U, 0x95959595U, 0xe4e4e4e4U, + 0x79797979U, 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, + 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, 0x6c6c6c6cU, + 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, 0x65656565U, 0x7a7a7a7aU, + 0xaeaeaeaeU, 0x08080808U, 0xbabababaU, 0x78787878U, 0x25252525U, + 0x2e2e2e2eU, 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, + 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, 0x4b4b4b4bU, + 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, 0x70707070U, 0x3e3e3e3eU, + 0xb5b5b5b5U, 0x66666666U, 0x48484848U, 0x03030303U, 0xf6f6f6f6U, + 0x0e0e0e0eU, 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, + 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, 0xe1e1e1e1U, + 0xf8f8f8f8U, 0x98989898U, 0x11111111U, 0x69696969U, 0xd9d9d9d9U, + 0x8e8e8e8eU, 0x94949494U, 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, + 0xe9e9e9e9U, 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, + 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, 0xbfbfbfbfU, + 0xe6e6e6e6U, 0x42424242U, 0x68686868U, 0x41414141U, 0x99999999U, + 0x2d2d2d2dU, 0x0f0f0f0fU, 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, + 0x16161616U, +}; +static const u32 Td0[256] = { + 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U, 0x3bab6bcbU, + 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U, 0x2030fa55U, 0xad766df6U, + 0x88cc7691U, 0xf5024c25U, 0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, + 0xb562a38fU, 0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U, + 0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U, 0x038f5fe7U, + 0x15929c95U, 0xbf6d7aebU, 0x955259daU, 0xd4be832dU, 0x587421d3U, + 0x49e06929U, 0x8ec9c844U, 0x75c2896aU, 0xf48e7978U, 0x99583e6bU, + 0x27b971ddU, 0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U, + 0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U, 0xb16477e0U, + 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U, 0x70486858U, 0x8f45fd19U, + 0x94de6c87U, 0x527bf8b7U, 0xab73d323U, 0x724b02e2U, 0xe31f8f57U, + 0x6655ab2aU, 0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U, + 0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU, 0x8acf1c2bU, + 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U, 0x65daf4cdU, 0x0605bed5U, + 0xd134621fU, 0xc4a6fe8aU, 0x342e539dU, 0xa2f355a0U, 0x058ae132U, + 0xa4f6eb75U, 0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U, + 0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U, 0x91548db5U, + 0x71c45d05U, 0x0406d46fU, 0x605015ffU, 0x1998fb24U, 0xd6bde997U, + 0x894043ccU, 0x67d99e77U, 0xb0e842bdU, 0x07898b88U, 0xe7195b38U, + 0x79c8eedbU, 0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U, + 0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU, 0xfd0efffbU, + 0x0f853856U, 0x3daed51eU, 0x362d3927U, 0x0a0fd964U, 0x685ca621U, + 0x9b5b54d1U, 0x24362e3aU, 0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, + 0x1b9b919eU, 0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U, + 0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU, 0x0e090d0bU, + 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U, 0x57f11985U, 0xaf75074cU, + 0xee99ddbbU, 0xa37f60fdU, 0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, + 0x5bfb7e34U, 0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U, + 0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U, 0x854a247dU, + 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU, 0x1d9e2f4bU, 0xdcb230f3U, + 0x0d8652ecU, 0x77c1e3d0U, 0x2bb3166cU, 0xa970b999U, 0x119448faU, + 0x47e96422U, 0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU, + 0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U, 0xa6f581cfU, + 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U, 0x2c3a9de4U, 0x5078920dU, + 0x6a5fcc9bU, 0x547e4662U, 0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, + 0x82c3aff5U, 0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U, + 0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU, 0xcd267809U, + 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U, 0xe6956e65U, 0xaaffe67eU, + 0x21bccf08U, 0xef15e8e6U, 0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, + 0x29b07cd6U, 0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U, + 0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U, 0xf104984aU, + 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU, 0x764dd68dU, 0x43efb04dU, + 0xccaa4d54U, 0xe49604dfU, 0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, + 0x4665517fU, 0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU, + 0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U, 0x9ad7618cU, + 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U, 0xcea927eeU, 0xb761c935U, + 0xe11ce5edU, 0x7a47b13cU, 0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, + 0x73c737bfU, 0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U, + 0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU, 0x161dc372U, + 0xbce2250cU, 0x283c498bU, 0xff0d9541U, 0x39a80171U, 0x080cb3deU, + 0xd8b4e49cU, 0x6456c190U, 0x7bcb8461U, 0xd532b670U, 0x486c5c74U, + 0xd0b85742U, +}; +static const u32 Td1[256] = { + 0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU, 0xcb3bab6bU, + 0xf11f9d45U, 0xabacfa58U, 0x934be303U, 0x552030faU, 0xf6ad766dU, + 0x9188cc76U, 0x25f5024cU, 0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, + 0x8fb562a3U, 0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U, + 0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U, 0xe7038f5fU, + 0x9515929cU, 0xebbf6d7aU, 0xda955259U, 0x2dd4be83U, 0xd3587421U, + 0x2949e069U, 0x448ec9c8U, 0x6a75c289U, 0x78f48e79U, 0x6b99583eU, + 0xdd27b971U, 0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU, + 0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU, 0xe0b16477U, + 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU, 0x58704868U, 0x198f45fdU, + 0x8794de6cU, 0xb7527bf8U, 0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, + 0x2a6655abU, 0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U, + 0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U, 0x2b8acf1cU, + 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U, 0xcd65daf4U, 0xd50605beU, + 0x1fd13462U, 0x8ac4a6feU, 0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, + 0x75a4f6ebU, 0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U, + 0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU, 0xb591548dU, + 0x0571c45dU, 0x6f0406d4U, 0xff605015U, 0x241998fbU, 0x97d6bde9U, + 0xcc894043U, 0x7767d99eU, 0xbdb0e842U, 0x8807898bU, 0x38e7195bU, + 0xdb79c8eeU, 0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U, + 0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U, 0xfbfd0effU, + 0x560f8538U, 0x1e3daed5U, 0x27362d39U, 0x640a0fd9U, 0x21685ca6U, + 0xd19b5b54U, 0x3a24362eU, 0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, + 0x9e1b9b91U, 0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU, + 0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U, 0x0b0e090dU, + 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U, 0x8557f119U, 0x4caf7507U, + 0xbbee99ddU, 0xfda37f60U, 0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, + 0x345bfb7eU, 0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U, + 0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U, 0x7d854a24U, + 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U, 0x4b1d9e2fU, 0xf3dcb230U, + 0xec0d8652U, 0xd077c1e3U, 0x6c2bb316U, 0x99a970b9U, 0xfa119448U, + 0x2247e964U, 0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U, + 0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU, 0xcfa6f581U, + 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU, 0xe42c3a9dU, 0x0d507892U, + 0x9b6a5fccU, 0x62547e46U, 0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, + 0xf582c3afU, 0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U, + 0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU, 0x09cd2678U, + 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU, 0x65e6956eU, 0x7eaaffe6U, + 0x0821bccfU, 0xe6ef15e8U, 0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, + 0xd629b07cU, 0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U, + 0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U, 0x4af10498U, + 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U, 0x8d764dd6U, 0x4d43efb0U, + 0x54ccaa4dU, 0xdfe49604U, 0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, + 0x7f466551U, 0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U, + 0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U, 0x8c9ad761U, + 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU, 0xeecea927U, 0x35b761c9U, + 0xede11ce5U, 0x3c7a47b1U, 0x599cd2dfU, 0x3f55f273U, 0x791814ceU, + 0xbf73c737U, 0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU, + 0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U, 0x72161dc3U, + 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U, 0x7139a801U, 0xde080cb3U, + 0x9cd8b4e4U, 0x906456c1U, 0x617bcb84U, 0x70d532b6U, 0x74486c5cU, + 0x42d0b857U, +}; +static const u32 Td2[256] = { + 0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U, 0x6bcb3babU, + 0x45f11f9dU, 0x58abacfaU, 0x03934be3U, 0xfa552030U, 0x6df6ad76U, + 0x769188ccU, 0x4c25f502U, 0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, + 0xa38fb562U, 0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU, + 0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U, 0x5fe7038fU, + 0x9c951592U, 0x7aebbf6dU, 0x59da9552U, 0x832dd4beU, 0x21d35874U, + 0x692949e0U, 0xc8448ec9U, 0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, + 0x71dd27b9U, 0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU, + 0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U, 0x77e0b164U, + 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U, 0x68587048U, 0xfd198f45U, + 0x6c8794deU, 0xf8b7527bU, 0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, + 0xab2a6655U, 0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U, + 0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U, 0x1c2b8acfU, + 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U, 0xf4cd65daU, 0xbed50605U, + 0x621fd134U, 0xfe8ac4a6U, 0x539d342eU, 0x55a0a2f3U, 0xe132058aU, + 0xeb75a4f6U, 0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU, + + 0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U, 0x8db59154U, + 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U, 0xfb241998U, 0xe997d6bdU, + 0x43cc8940U, 0x9e7767d9U, 0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, + 0xeedb79c8U, 0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U, + 0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU, 0xfffbfd0eU, + 0x38560f85U, 0xd51e3daeU, 0x3927362dU, 0xd9640a0fU, 0xa621685cU, + 0x54d19b5bU, 0x2e3a2436U, 0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, + 0x919e1b9bU, 0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U, + 0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU, 0x0d0b0e09U, + 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU, 0x198557f1U, 0x074caf75U, + 0xddbbee99U, 0x60fda37fU, 0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, + 0x7e345bfbU, 0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U, + 0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U, 0x247d854aU, + 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U, 0x2f4b1d9eU, 0x30f3dcb2U, + 0x52ec0d86U, 0xe3d077c1U, 0x166c2bb3U, 0xb999a970U, 0x48fa1194U, + 0x642247e9U, 0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U, + 0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U, 0x81cfa6f5U, + 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU, 0x9de42c3aU, 0x920d5078U, + 0xcc9b6a5fU, 0x4662547eU, 0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, + 0xaff582c3U, 0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U, + 0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU, 0x7809cd26U, + 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU, 0x6e65e695U, 0xe67eaaffU, + 0xcf0821bcU, 0xe8e6ef15U, 0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, + 0x7cd629b0U, 0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U, + 0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U, 0x984af104U, + 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U, 0xd68d764dU, 0xb04d43efU, + 0x4d54ccaaU, 0x04dfe496U, 0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, + 0x517f4665U, 0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU, + 0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U, 0x618c9ad7U, + 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U, 0x27eecea9U, 0xc935b761U, + 0xe5ede11cU, 0xb13c7a47U, 0xdf599cd2U, 0x733f55f2U, 0xce791814U, + 0x37bf73c7U, 0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U, + 0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U, 0xc372161dU, + 0x250cbce2U, 0x498b283cU, 0x9541ff0dU, 0x017139a8U, 0xb3de080cU, + 0xe49cd8b4U, 0xc1906456U, 0x84617bcbU, 0xb670d532U, 0x5c74486cU, + 0x5742d0b8U, +}; +static const u32 Td3[256] = { + 0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU, 0xab6bcb3bU, + 0x9d45f11fU, 0xfa58abacU, 0xe303934bU, 0x30fa5520U, 0x766df6adU, + 0xcc769188U, 0x024c25f5U, 0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, + 0x62a38fb5U, 0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU, + 0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU, 0x8f5fe703U, + 0x929c9515U, 0x6d7aebbfU, 0x5259da95U, 0xbe832dd4U, 0x7421d358U, + 0xe0692949U, 0xc9c8448eU, 0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, + 0xb971dd27U, 0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU, + 0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U, 0x6477e0b1U, + 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U, 0x48685870U, 0x45fd198fU, + 0xde6c8794U, 0x7bf8b752U, 0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, + 0x55ab2a66U, 0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U, + 0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU, 0xcf1c2b8aU, + 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU, 0xdaf4cd65U, 0x05bed506U, + 0x34621fd1U, 0xa6fe8ac4U, 0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, + 0xf6eb75a4U, 0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU, + 0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU, 0x548db591U, + 0xc45d0571U, 0x06d46f04U, 0x5015ff60U, 0x98fb2419U, 0xbde997d6U, + 0x4043cc89U, 0xd99e7767U, 0xe842bdb0U, 0x898b8807U, 0x195b38e7U, + 0xc8eedb79U, 0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U, + 0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU, 0x0efffbfdU, + 0x8538560fU, 0xaed51e3dU, 0x2d392736U, 0x0fd9640aU, 0x5ca62168U, + 0x5b54d19bU, 0x362e3a24U, 0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, + 0x9b919e1bU, 0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU, + 0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U, 0x090d0b0eU, + 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U, 0xf1198557U, 0x75074cafU, + 0x99ddbbeeU, 0x7f60fda3U, 0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, + 0xfb7e345bU, 0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U, + 0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U, 0x4a247d85U, + 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U, 0x9e2f4b1dU, 0xb230f3dcU, + 0x8652ec0dU, 0xc1e3d077U, 0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, + 0xe9642247U, 0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U, + 0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U, 0xf581cfa6U, + 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU, 0x3a9de42cU, 0x78920d50U, + 0x5fcc9b6aU, 0x7e466254U, 0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, + 0xc3aff582U, 0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU, + 0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU, 0x267809cdU, + 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U, 0x956e65e6U, 0xffe67eaaU, + 0xbccf0821U, 0x15e8e6efU, 0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, + 0xb07cd629U, 0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U, + 0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U, 0x04984af1U, + 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U, 0x4dd68d76U, 0xefb04d43U, + 0xaa4d54ccU, 0x9604dfe4U, 0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, + 0x65517f46U, 0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU, + 0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU, 0xd7618c9aU, + 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU, 0xa927eeceU, 0x61c935b7U, + 0x1ce5ede1U, 0x47b13c7aU, 0xd2df599cU, 0xf2733f55U, 0x14ce7918U, + 0xc737bf73U, 0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U, + 0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U, 0x1dc37216U, + 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU, 0xa8017139U, 0x0cb3de08U, + 0xb4e49cd8U, 0x56c19064U, 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, + 0xb85742d0U, +}; +static const u32 Td4[256] = { + 0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U, 0x30303030U, + 0x36363636U, 0xa5a5a5a5U, 0x38383838U, 0xbfbfbfbfU, 0x40404040U, + 0xa3a3a3a3U, 0x9e9e9e9eU, 0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, + 0xfbfbfbfbU, 0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U, + 0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U, 0x34343434U, + 0x8e8e8e8eU, 0x43434343U, 0x44444444U, 0xc4c4c4c4U, 0xdedededeU, + 0xe9e9e9e9U, 0xcbcbcbcbU, 0x54545454U, 0x7b7b7b7bU, 0x94949494U, + 0x32323232U, 0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU, + 0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU, 0x42424242U, + 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU, 0x08080808U, 0x2e2e2e2eU, + 0xa1a1a1a1U, 0x66666666U, 0x28282828U, 0xd9d9d9d9U, 0x24242424U, + 0xb2b2b2b2U, 0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U, + 0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U, 0x72727272U, + 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U, 0x86868686U, 0x68686868U, + 0x98989898U, 0x16161616U, 0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, + 0xccccccccU, 0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U, + 0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U, 0xfdfdfdfdU, + 0xededededU, 0xb9b9b9b9U, 0xdadadadaU, 0x5e5e5e5eU, 0x15151515U, + 0x46464646U, 0x57575757U, 0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, + 0x84848484U, 0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U, + 0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU, 0xf7f7f7f7U, + 0xe4e4e4e4U, 0x58585858U, 0x05050505U, 0xb8b8b8b8U, 0xb3b3b3b3U, + 0x45454545U, 0x06060606U, 0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, + 0x8f8f8f8fU, 0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U, + 0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U, 0x01010101U, + 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU, 0x3a3a3a3aU, 0x91919191U, + 0x11111111U, 0x41414141U, 0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, + 0xeaeaeaeaU, 0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU, + 0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U, 0x96969696U, + 0xacacacacU, 0x74747474U, 0x22222222U, 0xe7e7e7e7U, 0xadadadadU, + 0x35353535U, 0x85858585U, 0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, + 0xe8e8e8e8U, 0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU, + 0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U, 0x1d1d1d1dU, + 0x29292929U, 0xc5c5c5c5U, 0x89898989U, 0x6f6f6f6fU, 0xb7b7b7b7U, + 0x62626262U, 0x0e0e0e0eU, 0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, + 0x1b1b1b1bU, 0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU, + 0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U, 0x9a9a9a9aU, + 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU, 0x78787878U, 0xcdcdcdcdU, + 0x5a5a5a5aU, 0xf4f4f4f4U, 0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, + 0x33333333U, 0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U, + 0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U, 0x27272727U, + 0x80808080U, 0xececececU, 0x5f5f5f5fU, 0x60606060U, 0x51515151U, + 0x7f7f7f7fU, 0xa9a9a9a9U, 0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, + 0x0d0d0d0dU, 0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU, + 0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU, 0xa0a0a0a0U, + 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU, 0xaeaeaeaeU, 0x2a2a2a2aU, + 0xf5f5f5f5U, 0xb0b0b0b0U, 0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, + 0x3c3c3c3cU, 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U, + 0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU, 0xbabababaU, + 0x77777777U, 0xd6d6d6d6U, 0x26262626U, 0xe1e1e1e1U, 0x69696969U, + 0x14141414U, 0x63636363U, 0x55555555U, 0x21212121U, 0x0c0c0c0cU, + 0x7d7d7d7dU, +}; +static const u32 rcon[] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, 0x10000000, 0x20000000, + 0x40000000, 0x80000000, 0x1B000000, 0x36000000, /* for 128-bit blocks, + Rijndael never uses + more than 10 rcon + values */ +}; + +#define SWAP(x) (_lrotl(x, 8) & 0x00ff00ff | _lrotr(x, 8) & 0xff00ff00) + +#ifdef _MSC_VER +#define GETU32(p) SWAP(*((u32 *)(p))) +#define PUTU32(ct, st) \ + { \ + *((u32 *)(ct)) = SWAP((st)); \ + } +#else +#define GETU32(pt) \ + (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] << 8) ^ \ + ((u32)(pt)[3])) +#define PUTU32(ct, st) \ + { \ + (ct)[0] = (u8)((st) >> 24); \ + (ct)[1] = (u8)((st) >> 16); \ + (ct)[2] = (u8)((st) >> 8); \ + (ct)[3] = (u8)(st); \ + } +#endif + +/** + * Expand the cipher key into the encryption key schedule. + * + * @return the number of rounds for the given cipher key size. + */ +int rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits) +{ + int i = 0; + u32 temp; + + rk[0] = GETU32(cipherKey); + rk[1] = GETU32(cipherKey + 4); + rk[2] = GETU32(cipherKey + 8); + rk[3] = GETU32(cipherKey + 12); + if (keyBits == 128) { + for (;;) { + temp = rk[3]; + rk[4] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp)&0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[5] = rk[1] ^ rk[4]; + rk[6] = rk[2] ^ rk[5]; + rk[7] = rk[3] ^ rk[6]; + if (++i == 10) { + return 10; + } + rk += 4; + } + } + rk[4] = GETU32(cipherKey + 16); + rk[5] = GETU32(cipherKey + 20); + if (keyBits == 192) { + for (;;) { + temp = rk[5]; + rk[6] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp)&0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[7] = rk[1] ^ rk[6]; + rk[8] = rk[2] ^ rk[7]; + rk[9] = rk[3] ^ rk[8]; + if (++i == 8) { + return 12; + } + rk[10] = rk[4] ^ rk[9]; + rk[11] = rk[5] ^ rk[10]; + rk += 6; + } + } + rk[6] = GETU32(cipherKey + 24); + rk[7] = GETU32(cipherKey + 28); + if (keyBits == 256) { + for (;;) { + temp = rk[7]; + rk[8] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp)&0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[9] = rk[1] ^ rk[8]; + rk[10] = rk[2] ^ rk[9]; + rk[11] = rk[3] ^ rk[10]; + if (++i == 7) { + return 14; + } + temp = rk[11]; + rk[12] = rk[4] ^ (Te4[(temp >> 24)] & 0xff000000) ^ + (Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(temp >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(temp)&0xff] & 0x000000ff); + rk[13] = rk[5] ^ rk[12]; + rk[14] = rk[6] ^ rk[13]; + rk[15] = rk[7] ^ rk[14]; + + rk += 8; + } + } + return 0; +} + +/** + * Expand the cipher key into the decryption key schedule. + * + * @return the number of rounds for the given cipher key size. + */ +int rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits) +{ + int Nr, i, j; + u32 temp; + + /* expand the cipher key: */ + Nr = rijndaelKeySetupEnc(rk, cipherKey, keyBits); + /* invert the order of the round keys: */ + for (i = 0, j = 4 * Nr; i < j; i += 4, j -= 4) { + temp = rk[i]; + rk[i] = rk[j]; + rk[j] = temp; + temp = rk[i + 1]; + rk[i + 1] = rk[j + 1]; + rk[j + 1] = temp; + temp = rk[i + 2]; + rk[i + 2] = rk[j + 2]; + rk[j + 2] = temp; + temp = rk[i + 3]; + rk[i + 3] = rk[j + 3]; + rk[j + 3] = temp; + } + /* apply the inverse MixColumn transform to all round keys but the first + * and the last: */ + for (i = 1; i < Nr; i++) { + rk += 4; + rk[0] = Td0[Te4[(rk[0] >> 24)] & 0xff] ^ + Td1[Te4[(rk[0] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[0] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[0]) & 0xff] & 0xff]; + rk[1] = Td0[Te4[(rk[1] >> 24)] & 0xff] ^ + Td1[Te4[(rk[1] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[1] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[1]) & 0xff] & 0xff]; + rk[2] = Td0[Te4[(rk[2] >> 24)] & 0xff] ^ + Td1[Te4[(rk[2] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[2] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[2]) & 0xff] & 0xff]; + rk[3] = Td0[Te4[(rk[3] >> 24)] & 0xff] ^ + Td1[Te4[(rk[3] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[3] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[3]) & 0xff] & 0xff]; + } + return Nr; +} + +void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], + u8 ct[16]) +{ + u32 s0, s1, s2, s3, t0, t1, t2, t3; +#ifndef FULL_UNROLL + int r; +#endif /* ?FULL_UNROLL */ + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(pt) ^ rk[0]; + s1 = GETU32(pt + 4) ^ rk[1]; + s2 = GETU32(pt + 8) ^ rk[2]; + s3 = GETU32(pt + 12) ^ rk[3]; +#ifdef FULL_UNROLL + /* round 1: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[4]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[5]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[6]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[7]; + /* round 2: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[8]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[9]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[10]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[11]; + /* round 3: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[12]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[13]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[14]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[15]; + /* round 4: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[16]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[17]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[18]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[19]; + /* round 5: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[20]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[21]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[22]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[23]; + /* round 6: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[24]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[25]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[26]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[27]; + /* round 7: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[28]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[29]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[30]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[31]; + /* round 8: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[32]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[33]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[34]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[35]; + /* round 9: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[36]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[37]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[38]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[39]; + if (Nr > 10) { + /* round 10: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ + Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[40]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ + Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[41]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ + Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[42]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ + Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[43]; + /* round 11: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[44]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[45]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[46]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[47]; + if (Nr > 12) { + /* round 12: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ + Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[48]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ + Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[49]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ + Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[50]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ + Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[51]; + /* round 13: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[52]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[53]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[54]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[55]; + } + } + rk += Nr << 2; +#else /* !FULL_UNROLL */ + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + t0 = Te0[(s0 >> 24)] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[(s3)&0xff] ^ rk[4]; + t1 = Te0[(s1 >> 24)] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[(s0)&0xff] ^ rk[5]; + t2 = Te0[(s2 >> 24)] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[(s1)&0xff] ^ rk[6]; + t3 = Te0[(s3 >> 24)] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[(s2)&0xff] ^ rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + s0 = Te0[(t0 >> 24)] ^ Te1[(t1 >> 16) & 0xff] ^ + Te2[(t2 >> 8) & 0xff] ^ Te3[(t3)&0xff] ^ rk[0]; + s1 = Te0[(t1 >> 24)] ^ Te1[(t2 >> 16) & 0xff] ^ + Te2[(t3 >> 8) & 0xff] ^ Te3[(t0)&0xff] ^ rk[1]; + s2 = Te0[(t2 >> 24)] ^ Te1[(t3 >> 16) & 0xff] ^ + Te2[(t0 >> 8) & 0xff] ^ Te3[(t1)&0xff] ^ rk[2]; + s3 = Te0[(t3 >> 24)] ^ Te1[(t0 >> 16) & 0xff] ^ + Te2[(t1 >> 8) & 0xff] ^ Te3[(t2)&0xff] ^ rk[3]; + } +#endif /* ?FULL_UNROLL */ + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = (Te4[(t0 >> 24)] & 0xff000000) ^ + (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t3)&0xff] & 0x000000ff) ^ rk[0]; + PUTU32(ct, s0); + s1 = (Te4[(t1 >> 24)] & 0xff000000) ^ + (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t0)&0xff] & 0x000000ff) ^ rk[1]; + PUTU32(ct + 4, s1); + s2 = (Te4[(t2 >> 24)] & 0xff000000) ^ + (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t1)&0xff] & 0x000000ff) ^ rk[2]; + PUTU32(ct + 8, s2); + s3 = (Te4[(t3 >> 24)] & 0xff000000) ^ + (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t2)&0xff] & 0x000000ff) ^ rk[3]; + PUTU32(ct + 12, s3); +} + +void rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16], + u8 pt[16]) +{ + u32 s0, s1, s2, s3, t0, t1, t2, t3; +#ifndef FULL_UNROLL + int r; +#endif /* ?FULL_UNROLL */ + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(ct) ^ rk[0]; + s1 = GETU32(ct + 4) ^ rk[1]; + s2 = GETU32(ct + 8) ^ rk[2]; + s3 = GETU32(ct + 12) ^ rk[3]; +#ifdef FULL_UNROLL + /* round 1: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[4]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[5]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[6]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[7]; + /* round 2: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[8]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[9]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[10]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[11]; + /* round 3: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[12]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[13]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[14]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[15]; + /* round 4: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[16]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[17]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[18]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[19]; + /* round 5: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[20]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[21]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[22]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[23]; + /* round 6: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[24]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[25]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[26]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[27]; + /* round 7: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[28]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[29]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[30]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[31]; + /* round 8: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[32]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[33]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[34]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[35]; + /* round 9: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[36]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[37]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[38]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[39]; + if (Nr > 10) { + /* round 10: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ + Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[40]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ + Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[41]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ + Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[42]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ + Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[43]; + /* round 11: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[44]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[45]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[46]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[47]; + if (Nr > 12) { + /* round 12: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ + Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[48]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ + Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[49]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ + Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[50]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ + Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[51]; + /* round 13: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[52]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[53]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[54]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[55]; + } + } + rk += Nr << 2; +#else /* !FULL_UNROLL */ + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + t0 = Td0[(s0 >> 24)] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[(s1)&0xff] ^ rk[4]; + t1 = Td0[(s1 >> 24)] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[(s2)&0xff] ^ rk[5]; + t2 = Td0[(s2 >> 24)] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[(s3)&0xff] ^ rk[6]; + t3 = Td0[(s3 >> 24)] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[(s0)&0xff] ^ rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + s0 = Td0[(t0 >> 24)] ^ Td1[(t3 >> 16) & 0xff] ^ + Td2[(t2 >> 8) & 0xff] ^ Td3[(t1)&0xff] ^ rk[0]; + s1 = Td0[(t1 >> 24)] ^ Td1[(t0 >> 16) & 0xff] ^ + Td2[(t3 >> 8) & 0xff] ^ Td3[(t2)&0xff] ^ rk[1]; + s2 = Td0[(t2 >> 24)] ^ Td1[(t1 >> 16) & 0xff] ^ + Td2[(t0 >> 8) & 0xff] ^ Td3[(t3)&0xff] ^ rk[2]; + s3 = Td0[(t3 >> 24)] ^ Td1[(t2 >> 16) & 0xff] ^ + Td2[(t1 >> 8) & 0xff] ^ Td3[(t0)&0xff] ^ rk[3]; + } +#endif /* ?FULL_UNROLL */ + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = (Td4[(t0 >> 24)] & 0xff000000) ^ + (Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(t1)&0xff] & 0x000000ff) ^ rk[0]; + PUTU32(pt, s0); + s1 = (Td4[(t1 >> 24)] & 0xff000000) ^ + (Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(t2)&0xff] & 0x000000ff) ^ rk[1]; + PUTU32(pt + 4, s1); + s2 = (Td4[(t2 >> 24)] & 0xff000000) ^ + (Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(t3)&0xff] & 0x000000ff) ^ rk[2]; + PUTU32(pt + 8, s2); + s3 = (Td4[(t3 >> 24)] & 0xff000000) ^ + (Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(t0)&0xff] & 0x000000ff) ^ rk[3]; + PUTU32(pt + 12, s3); +} + +#ifdef INTERMEDIATE_VALUE_KAT + +void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds) +{ + int r; + u32 s0, s1, s2, s3, t0, t1, t2, t3; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(block) ^ rk[0]; + s1 = GETU32(block + 4) ^ rk[1]; + s2 = GETU32(block + 8) ^ rk[2]; + s3 = GETU32(block + 12) ^ rk[3]; + rk += 4; + + /* + * Nr - 1 full rounds: + */ + for (r = (rounds < Nr ? rounds : Nr - 1); r > 0; r--) { + t0 = Te0[(s0 >> 24)] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[(s3)&0xff] ^ rk[0]; + t1 = Te0[(s1 >> 24)] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[(s0)&0xff] ^ rk[1]; + t2 = Te0[(s2 >> 24)] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[(s1)&0xff] ^ rk[2]; + t3 = Te0[(s3 >> 24)] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[(s2)&0xff] ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + rk += 4; + } + + /* + * apply last round and + * map cipher state to byte array block: + */ + if (rounds == Nr) { + t0 = (Te4[(s0 >> 24)] & 0xff000000) ^ + (Te4[(s1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s2 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s3)&0xff] & 0x000000ff) ^ rk[0]; + t1 = (Te4[(s1 >> 24)] & 0xff000000) ^ + (Te4[(s2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s3 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s0)&0xff] & 0x000000ff) ^ rk[1]; + t2 = (Te4[(s2 >> 24)] & 0xff000000) ^ + (Te4[(s3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s0 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s1)&0xff] & 0x000000ff) ^ rk[2]; + t3 = (Te4[(s3 >> 24)] & 0xff000000) ^ + (Te4[(s0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s1 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s2)&0xff] & 0x000000ff) ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + } + + PUTU32(block, s0); + PUTU32(block + 4, s1); + PUTU32(block + 8, s2); + PUTU32(block + 12, s3); +} + +void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds) +{ + int r; + u32 s0, s1, s2, s3, t0, t1, t2, t3; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(block) ^ rk[0]; + s1 = GETU32(block + 4) ^ rk[1]; + s2 = GETU32(block + 8) ^ rk[2]; + s3 = GETU32(block + 12) ^ rk[3]; + rk += 4; + + /* + * Nr - 1 full rounds: + */ + for (r = (rounds < Nr ? rounds : Nr) - 1; r > 0; r--) { + t0 = Td0[(s0 >> 24)] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[(s1)&0xff] ^ rk[0]; + t1 = Td0[(s1 >> 24)] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[(s2)&0xff] ^ rk[1]; + t2 = Td0[(s2 >> 24)] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[(s3)&0xff] ^ rk[2]; + t3 = Td0[(s3 >> 24)] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[(s0)&0xff] ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + rk += 4; + } + + /* + * complete the last round and + * map cipher state to byte array block: + */ + t0 = (Td4[(s0 >> 24)] & 0xff000000) ^ + (Td4[(s3 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s2 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(s1)&0xff] & 0x000000ff); + t1 = (Td4[(s1 >> 24)] & 0xff000000) ^ + (Td4[(s0 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s3 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(s2)&0xff] & 0x000000ff); + t2 = (Td4[(s2 >> 24)] & 0xff000000) ^ + (Td4[(s1 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s0 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(s3)&0xff] & 0x000000ff); + t3 = (Td4[(s3 >> 24)] & 0xff000000) ^ + (Td4[(s2 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s1 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(s0)&0xff] & 0x000000ff); + + if (rounds == Nr) { + t0 ^= rk[0]; + t1 ^= rk[1]; + t2 ^= rk[2]; + t3 ^= rk[3]; + } + + PUTU32(block, t0); + PUTU32(block + 4, t1); + PUTU32(block + 8, t2); + PUTU32(block + 12, t3); +} + +#endif /* INTERMEDIATE_VALUE_KAT */ diff --git a/lib/rijndael-alg-fst.h b/lib/rijndael-alg-fst.h new file mode 100644 index 0000000..eabff49 --- /dev/null +++ b/lib/rijndael-alg-fst.h @@ -0,0 +1,53 @@ +/** + * rijndael-alg-fst.h + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen + * @author Antoon Bosselaers + * @author Paulo Barreto + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef RIJNDAEL_ALG_FST_H +#define RIJNDAEL_ALG_FST_H + +#define MAXKC (256 / 32) +#define MAXKB (256 / 8) +#define MAXNR 14 + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; + +int rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits); +int rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits); +void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], + u8 ct[16]); +void rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16], + u8 pt[16]); + +#ifdef INTERMEDIATE_VALUE_KAT +void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds); +void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds); +#endif /* INTERMEDIATE_VALUE_KAT */ + +#endif /* RIJNDAEL_ALG_FST_H */ diff --git a/lib/types.h b/lib/types.h new file mode 100644 index 0000000..ef4ec2c --- /dev/null +++ b/lib/types.h @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_TYPES_H +#define ZMAP_TYPES_H + +#include + +#include + +typedef uint32_t ipaddr_n_t; // IPv4 address network order +typedef uint32_t ipaddr_h_t; // IPv4 address host order +typedef uint16_t port_n_t; // port network order +typedef uint16_t port_h_t; // port host order +typedef unsigned char macaddr_t; + +#endif /* ZMAP_TYPES_H */ diff --git a/lib/util.c b/lib/util.c new file mode 100644 index 0000000..bbe6abc --- /dev/null +++ b/lib/util.c @@ -0,0 +1,371 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE +#include "util.h" + +#include +#include +#include +#include + +#include "includes.h" +#include "xalloc.h" + +#include +#include +#include +#include +#include +#include + +#include "../lib/logger.h" + +#define MAX_SPLITS 128 + +int max_int(int a, int b) +{ + if (a >= b) { + return a; + } + return b; +} + +int min_int(int a, int b) +{ + if (a >= b) { + return b; + } + return a; +} + +void enforce_range(const char *name, int v, int min, int max) +{ + if (check_range(v, min, max) == EXIT_FAILURE) { + log_fatal("zmap", "argument `%s' must be between %d and %d\n", + name, min, max); + } +} + +void split_string(const char *in, int *len, const char ***results) +{ + const char **fields = xcalloc(MAX_SPLITS, sizeof(const char *)); + int retvlen = 0; + const char *currloc = in; + // parse csv into a set of strings + while (1) { + assert(retvlen < MAX_SPLITS); + size_t len = strcspn(currloc, ", "); + if (len == 0) { + currloc++; + } else { + char *new = xmalloc(len + 1); + strncpy(new, currloc, len); + new[len] = '\0'; + fields[retvlen++] = new; + assert(fields[retvlen - 1]); + } + if (len == strlen(currloc)) { + break; + } + currloc += len; + } + *results = fields; + *len = retvlen; +} + +void fprintw(FILE *f, const char *s, size_t w) +{ + if (strlen(s) <= w) { + fprintf(f, "%s", s); + return; + } + // process each line individually in order to + // respect existing line breaks in string. + char *news = strdup(s); + char *pch = strtok(news, "\n"); + while (pch) { + if (strlen(pch) <= w) { + printf("%s\n", pch); + pch = strtok(NULL, "\n"); + continue; + } + char *t = pch; + while (strlen(t)) { + size_t numchars = 0; // number of chars to print + char *tmp = t; + while (1) { + size_t new = strcspn(tmp, " ") + 1; + if (new == strlen(tmp) || new > w) { + // there are no spaces in the string, + // so, just print the entire thing on + // one line; + numchars += new; + break; + } else if (numchars + new > w) { + // if we added any more, we'd be over w + // chars so time to print the line and + // move on to the next. + break; + } else { + tmp += (size_t) new; + numchars += new; + } + } + fprintf(f, "%.*s\n", (int)numchars, t); + t += (size_t)numchars; + if (t > pch + (size_t)strlen(pch)) { + break; + } + } + pch = strtok(NULL, "\n"); + } + free(news); +} + +uint32_t parse_max_hosts(char *max_targets) +{ + char *end; + errno = 0; + double v = strtod(max_targets, &end); + if (end == max_targets || errno != 0) { + log_fatal("argparse", "can't convert max-targets to a number"); + } + if (end[0] == '%' && end[1] == '\0') { + // treat as percentage + v = v * ((unsigned long long int)1 << 32) / 100.; + } else if (end[0] != '\0') { + log_fatal("eargparse", "extra characters after max-targets"); + } + if (v <= 0) { + return 0; + } else if (v >= ((unsigned long long int)1 << 32)) { + return 0xFFFFFFFF; + } else { + return v; + } +} + +// pretty print elapsed (or estimated) number of seconds +void time_string(uint32_t time, int est, char *buf, size_t len) +{ + int y = time / 31556736; + int d = (time % 31556736) / 86400; + int h = (time % 86400) / 3600; + int m = (time % 3600) / 60; + int s = time % 60; + + if (est) { + if (y > 0) { + snprintf(buf, len, "%d years", y); + } else if (d > 9) { + snprintf(buf, len, "%dd", d); + } else if (d > 0) { + snprintf(buf, len, "%dd%02dh", d, h); + } else if (h > 9) { + snprintf(buf, len, "%dh", h); + } else if (h > 0) { + snprintf(buf, len, "%dh%02dm", h, m); + } else if (m > 9) { + snprintf(buf, len, "%dm", m); + } else if (m > 0) { + snprintf(buf, len, "%dm%02ds", m, s); + } else { + snprintf(buf, len, "%ds", s); + } + } else { + if (d > 0) { + snprintf(buf, len, "%dd%d:%02d:%02d", d, h, m, s); + } else if (h > 0) { + snprintf(buf, len, "%d:%02d:%02d", h, m, s); + } else { + snprintf(buf, len, "%d:%02d", m, s); + } + } +} + +// pretty print quantities +void number_string(uint32_t n, char *buf, size_t len) +{ + int figs = 0; + if (n < 1000) { + snprintf(buf, len, "%u ", n); + } else if (n < 1000000) { + if (n < 10000) { + figs = 2; + } else if (n < 100000) { + figs = 1; + } + snprintf(buf, len, "%0.*f K", figs, (float)n / 1000.); + } else { + if (figs < 10000000) { + figs = 2; + } else if (figs < 100000000) { + figs = 1; + } + snprintf(buf, len, "%0.*f M", figs, (float)n / 1000000.); + } +} + +int parse_mac(macaddr_t *out, char *in) +{ + if (strlen(in) < MAC_ADDR_LEN * 3 - 1) + return 0; + char octet[4]; + octet[2] = '\0'; + for (int i = 0; i < MAC_ADDR_LEN; i++) { + if (i < MAC_ADDR_LEN - 1 && in[i * 3 + 2] != ':') { + return 0; + } + strncpy(octet, &in[i * 3], 2); + char *err = NULL; + long b = strtol(octet, &err, 16); + if (err && *err != '\0') { + return 0; + } + out[i] = b & 0xFF; + } + return 1; +} + +int check_range(int v, int min, int max) +{ + if (v < min || v > max) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int file_exists(char *name) +{ + FILE *file = fopen(name, "r"); + if (!file) + return 0; + fclose(file); + return 1; +} + +#if defined(__APPLE__) +#include +#endif + +int drop_privs(void) +{ + struct passwd *pw; + if (geteuid() != 0) { + /* Not root */ + return EXIT_SUCCESS; + } + if ((pw = getpwnam("nobody")) != NULL) { + if (setuid(pw->pw_uid) == 0) { + return EXIT_SUCCESS; // success + } + } + return EXIT_FAILURE; +} + +#if defined(__APPLE__) + +#include + +int set_cpu(uint32_t core) +{ + mach_port_t tid = pthread_mach_thread_np(pthread_self()); + struct thread_affinity_policy policy; + policy.affinity_tag = core; + kern_return_t ret = thread_policy_set(tid, THREAD_AFFINITY_POLICY, + (thread_policy_t)&policy, + THREAD_AFFINITY_POLICY_COUNT); + if (ret != KERN_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#elif defined(__DragonFly__) + +#include + +int set_cpu(uint32_t core) +{ + if (usched_set(getpid(), USCHED_SET_CPU, &core, sizeof(core)) != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#elif defined(__NetBSD__) + +int set_cpu(uint32_t core) +{ + cpuset_t *cpuset = cpuset_create(); + if (cpuset == NULL) { + return EXIT_FAILURE; + } + cpuset_zero(cpuset); + cpuset_set(core, cpuset); + cpuset_destroy(cpuset); + + if (pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), + cpuset) != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#else + +#if defined(__FreeBSD__) +#include +#include +#include +#define cpu_set_t cpuset_t +#endif + +int set_cpu(uint32_t core) +{ + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(core, &cpuset); + + if (pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), + &cpuset) != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#endif + +double now(void) +{ + struct timeval now; + gettimeofday(&now, NULL); + return (double)now.tv_sec + (double)now.tv_usec / 1000000.; +} + +double steady_now(void) +{ +#if defined(_POSIX_TIMERS) && defined(_POSIX_MONOTONIC_CLOCK) + struct timespec tp; + clock_gettime(CLOCK_MONOTONIC, &tp); + return (double)tp.tv_sec + (double)tp.tv_nsec / 1000000000.; +#else + struct timeval now; + gettimeofday(&now, NULL); + return (double)now.tv_sec + (double)now.tv_usec / 1000000.; +#endif +} diff --git a/lib/util.h b/lib/util.h new file mode 100644 index 0000000..fea61f3 --- /dev/null +++ b/lib/util.h @@ -0,0 +1,74 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_UTIL_H +#define ZMAP_UTIL_H + +#include +#include +#include + +#include "types.h" + +int max_int(int a, int b); +int min_int(int a, int b); + +uint32_t parse_max_hosts(char *max_targets); +void enforce_range(const char *name, int v, int min, int max); + +// Splits comma delimited string into char*[]. Does not handle +// escaping or complicated setups - designed to process a set +// of fields that the user wants output +void split_string(const char *in, int *len, const char ***results); + +// Print a string using w length long lines, attempting to break on +// spaces +void fprintw(FILE *f, const char *s, size_t w); + +// pretty print elapsed (or estimated) number of seconds +void time_string(uint32_t time, int est, char *buf, size_t len); + +// pretty print quantities +void number_string(uint32_t n, char *buf, size_t len); + +// Convert a string representation of a MAC address to a byte array +int parse_mac(macaddr_t *out, char *in); + +int check_range(int v, int min, int max); + +int file_exists(char *name); + +// If running as root, drops privileges to that of user "nobody". +// Otherwise, does nothing. +int drop_privs(void); + +// Set CPU affinity to a single core +int set_cpu(uint32_t core); + +// The number of seconds and microseconds since the Epoch. +double now(void); + +// The number of seconds and nanoseconds since an unspecified point in time. +// On supported hosts, this value is guaranteed to never decrease. +// +// According to the POSIX specification the `clock_gettime` function is part +// of the Timers option and may not be available on all implementations. +// +// On hosts where a monotonic clock is not available, this falls back +// to `gettimeofday` which was ZMap's original implementation. +double steady_now(void); + +#endif /* ZMAP_UTIL_H */ diff --git a/lib/xalloc.c b/lib/xalloc.c new file mode 100644 index 0000000..dfb64df --- /dev/null +++ b/lib/xalloc.c @@ -0,0 +1,55 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "xalloc.h" +#include "logger.h" + +#include +#include + +static void die(void) __attribute__((noreturn)); + +void *xcalloc(size_t count, size_t size) +{ + void *res = calloc(count, size); + if (res == NULL) { + die(); + } + return res; +} + +void xfree(void *ptr) { free(ptr); } + +void *xmalloc(size_t size) +{ + void *res = malloc(size); + if (res == NULL) { + die(); + } + memset(res, 0, size); + return res; +} + +void *xrealloc(void *ptr, size_t size) +{ + void *res = realloc(ptr, size); + if (res == NULL) { + die(); + } + return res; +} + +void die(void) { log_fatal("zmap", "Out of memory"); } diff --git a/lib/xalloc.h b/lib/xalloc.h new file mode 100644 index 0000000..5dd6001 --- /dev/null +++ b/lib/xalloc.h @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_ALLOC_H +#define ZMAP_ALLOC_H + +#include + +void *xcalloc(size_t count, size_t size); + +void xfree(void *ptr); + +void *xmalloc(size_t size); + +void *xrealloc(void *ptr, size_t size); + +#endif /* ZMAP_ALLOC_H */ diff --git a/release-build.sh b/release-build.sh new file mode 100644 index 0000000..9c7f718 --- /dev/null +++ b/release-build.sh @@ -0,0 +1,7 @@ +#!/bin/bash +cmake \ + -DENABLE_DEVELOPMENT=off \ + -DZMAP_VERSION=$1 \ + -DENABLE_LOG_TRACE=off \ + . +make diff --git a/scripts/check_manfile.py b/scripts/check_manfile.py new file mode 100755 index 0000000..205b7da --- /dev/null +++ b/scripts/check_manfile.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# +# CI runs this script to verify that options appearing in ZTools' ggo.in files +# also appear in their .ronn files. It does not check that `make manpages` has +# actually been run. +# +# This script assumes it's being run from the root of the zmap repository. +# + +import sys + +checks = [ + ("zopt.ggo.in", "zmap.1.ronn"), + ("zbopt.ggo.in", "zblocklist.1.ronn"), + ("zitopt.ggo.in", "ziterate.1.ronn"), + ("ztopt.ggo.in", "ztee.1.ronn") +] + +failures = False + +for ggo, ronn in checks: + options = [] + with open("src/" + ggo) as fd: + for l in fd: + if l.startswith("option "): + option = l.split()[1].lstrip('"').rstrip('"') + options.append(option) + + man = open("src/" + ronn).read() + + for option in options: + if option not in man: + failures = True + sys.stderr.write(f"option \"{option}\" is present in \"{ggo}\" but missing from man file \"{ronn}\"\n") + sys.stderr.flush() + +if failures: + sys.exit(1) diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..87de78c --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,4 @@ +lexer.c +lexer.h +parser.c +parser.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..9b7dbcc --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,273 @@ +include_directories( + "${CMAKE_CURRENT_BINARY_DIR}" + ${PROJECT_SOURCE_DIR}/lib + ${PROJECT_SOURCE_DIR}/src + ${PROJECT_SOURCE_DIR}/src/output_modules + ${PROJECT_SOURCE_DIR}/src/tests +) + +# ADD YOUR PROBE MODULE HERE +set(EXTRA_PROBE_MODULES +) + +# ADD YOUR OUTPUT MODULE HERE +set(EXTRA_OUTPUT_MODULES +) + +set(OUTPUT_MODULE_SOURCES + output_modules/module_csv.c + output_modules/module_json.c + output_modules/output_modules.c +) + +set(PROBE_MODULE_SOURCES + probe_modules/module_icmp_echo.c + probe_modules/module_icmp_echo_time.c + probe_modules/module_tcp_synscan.c + probe_modules/module_tcp_synackscan.c + #probe_modules/module_tcp_cisco_backdoor.c + probe_modules/module_udp.c + probe_modules/packet.c + probe_modules/probe_modules.c + probe_modules/module_ntp.c + probe_modules/module_upnp.c + probe_modules/module_dns.c + probe_modules/module_bacnet.c +) + +set(SOURCES + aesrand.c + cyclic.c + expression.c + fieldset.c + filter.c + get_gateway.c + iterator.c + monitor.c + ports.c + recv.c + send.c + shard.c + socket.c + state.c + summary.c + utility.c + validate.c + zmap.c + zopt_compat.c + "${CMAKE_CURRENT_BINARY_DIR}/zopt.h" + "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" + "${CMAKE_CURRENT_BINARY_DIR}/parser.c" + ${EXTRA_PROBE_MODULES} + ${EXTRA_OUTPUT_MODULES} + ${PROBE_MODULE_SOURCES} + ${OUTPUT_MODULE_SOURCES} +) + +set(ZTESTSOURCES + aesrand.c + cyclic.c + expression.c + fieldset.c + filter.c + get_gateway.c + iterator.c + monitor.c + ports.c + recv.c + send.c + shard.c + socket.c + state.c + summary.c + validate.c + ztopt_compat.c + ${PROBE_MODULE_SOURCES} + ${OUTPUT_MODULE_SOURCES} + tests/test_harness.c + "${CMAKE_CURRENT_BINARY_DIR}/ztopt.h" + "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" + "${CMAKE_CURRENT_BINARY_DIR}/parser.c" + ${EXTRA_PROBE_MODULES} + ${EXTRA_OUTPUT_MODULES} +) + +set(ZBLSOURCES + zblocklist.c + zbopt_compat.c + "${CMAKE_CURRENT_BINARY_DIR}/zbopt.h" +) + +set(ZITSOURCES + aesrand.c + cyclic.c + iterator.c + ports.c + shard.c + state.c + validate.c + zitopt_compat.c + ziterate.c + "${CMAKE_CURRENT_BINARY_DIR}/zitopt.h" +) + +set(ZTEESOURCES + ztee.c + topt_compat.c + "${CMAKE_CURRENT_BINARY_DIR}/topt.h" +) + +# Handle various versions of socket +if(WITH_PFRING) + set(SOURCES ${SOURCES} socket-pfring.c) + set(ZTESTSOURCES ${ZTESTSOURCES} socket-pfring.c) +elseif (APPLE OR BSD) + set(SOURCES ${SOURCES} socket-bsd.c) + set(ZTESTSOURCES ${ZTESTSOURCES} socket-bsd.c) +else() + set(SOURCES ${SOURCES} socket-linux.c send-linux.c) + set(ZTESTSOURCES ${ZTESTSOURCES} socket-linux.c send-linux.c) +endif() + +# Handle various versions of recv +if(WITH_PFRING) + set(SOURCES ${SOURCES} recv-pfring.c) + set(ZTESTSOURCES ${ZTESTSOURCES} recv-pfring.c) +else() + set(SOURCES ${SOURCES} recv-pcap.c) + set(ZTESTSOURCES ${ZTESTSOURCES} recv-pcap.c) +endif() + +# Set configure time zmap version +configure_file(topt.ggo.in ${CMAKE_BINARY_DIR}/src/topt.ggo @ONLY) +configure_file(zbopt.ggo.in ${CMAKE_BINARY_DIR}/src/zbopt.ggo @ONLY) +configure_file(zitopt.ggo.in ${CMAKE_BINARY_DIR}/src/zitopt.ggo @ONLY) +configure_file(zopt.ggo.in ${CMAKE_BINARY_DIR}/src/zopt.ggo @ONLY) +configure_file(ztopt.ggo.in ${CMAKE_BINARY_DIR}/src/ztopt.ggo @ONLY) +# Additional ggo.in's should be added here and CMakeVersion.txt + +# This sets a *build* time dependency that checks git +if("${ZMAP_VERSION}" STREQUAL "DEVELOPMENT") + add_custom_target(git_versioning ALL + COMMAND ${CMAKE_COMMAND} -D ORIG_SRC_DIR:STRING="${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/src/CMakeVersion.txt" + ) +endif() + +add_custom_command(OUTPUT zopt.h + COMMAND gengetopt -C --no-help --no-version --unamed-opts=SUBNETS -i "${CMAKE_CURRENT_BINARY_DIR}/zopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/zopt.ggo" +) + +add_custom_command(OUTPUT topt.h + COMMAND gengetopt -S --no-help --no-version --unamed-opts=FILE -i "${CMAKE_CURRENT_BINARY_DIR}/topt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/topt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/topt.ggo" +) + +add_custom_command(OUTPUT zbopt.h + COMMAND gengetopt -C --no-help --no-version -i "${CMAKE_CURRENT_BINARY_DIR}/zbopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zbopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/zbopt.ggo" +) + +add_custom_command(OUTPUT zitopt.h + COMMAND gengetopt -C --no-help --no-version --unamed-opts=SUBNETS -i "${CMAKE_CURRENT_BINARY_DIR}/zitopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zitopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/zitopt.ggo" +) + +add_custom_command(OUTPUT ztopt.h + COMMAND gengetopt -C --no-help --no-version -i "${CMAKE_CURRENT_BINARY_DIR}/ztopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/ztopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/ztopt.ggo" +) + +add_custom_command(OUTPUT lexer.c + COMMAND flex -o "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" --header-file="${CMAKE_CURRENT_BINARY_DIR}/lexer.h" "${CMAKE_CURRENT_SOURCE_DIR}/lexer.l" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/lexer.l" +) + +if(NetBSD) +add_custom_command(OUTPUT parser.c + COMMAND yacc -d -o "${CMAKE_CURRENT_BINARY_DIR}/parser.c" "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" +) +else() +add_custom_command(OUTPUT parser.c + COMMAND byacc -d -o "${CMAKE_CURRENT_BINARY_DIR}/parser.c" "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" +) +endif() + +add_custom_target(manpages ronn "${CMAKE_CURRENT_SOURCE_DIR}/zmap.1.ronn" --organization="ZMap" --manual="zmap" + COMMAND ronn "${CMAKE_CURRENT_SOURCE_DIR}/zblocklist.1.ronn" --organization="ZMap" --manual="zblocklist" + COMMAND ronn "${CMAKE_CURRENT_SOURCE_DIR}/ziterate.1.ronn" --organization="ZMap" --manual="ziterate" + COMMAND ronn "${CMAKE_CURRENT_SOURCE_DIR}/ztee.1.ronn" --organization="ZMap" --manual="ztee" + SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/zblocklist.1.ronn" "${CMAKE_CURRENT_SOURCE_DIR}/ziterate.1.ronn" "${CMAKE_CURRENT_SOURCE_DIR}/zmap.1.ronn" "${CMAKE_CURRENT_SOURCE_DIR}/ztee.1.ronn" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" +) + +add_executable(zmap ${SOURCES}) +add_executable(zblocklist ${ZBLSOURCES}) +add_executable(ziterate ${ZITSOURCES}) +add_executable(ztee ${ZTEESOURCES}) +add_executable(ztests ${ZTESTSOURCES}) + +if(APPLE OR BSD) +else() + set(ZTESTSOURCES ${ZTESTSOURCES} send-linux.c) + set(SOURCES ${SOURCES} send-linux.c) +endif() + +target_link_libraries( + zmap + zmaplib + ${PFRING_LIBRARIES} + pcap gmp m unistring + ${JSON_LIBRARIES} + ${JUDY_LIBRARIES} +) + +target_link_libraries( + zblocklist + zmaplib + m +) + +target_link_libraries( + ziterate + zmaplib + gmp + m +) + +target_link_libraries( + ztee + zmaplib + m +) + +target_link_libraries( + ztests + zmaplib + ${PFRING_LIBRARIES} + pcap gmp m unistring + ${JSON_LIBRARIES} + ${JUDY_LIBRARIES} +) + +# Install binary +install( + TARGETS + zmap + zblocklist + ziterate + ztee + RUNTIME DESTINATION sbin +) + +# Install Manpages +install( + FILES + zmap.1 + zblocklist.1 + ziterate.1 + ztee.1 + DESTINATION share/man/man1 +) diff --git a/src/CMakeVersion.txt b/src/CMakeVersion.txt new file mode 100644 index 0000000..893bc2f --- /dev/null +++ b/src/CMakeVersion.txt @@ -0,0 +1,15 @@ +set(GIT_CMD "git") +set(GIT_ARGS "log" "-n" "1" "--pretty=format:%h - %ad") +execute_process(COMMAND ${GIT_CMD} ${GIT_ARGS} + RESULT_VARIABLE GIT_RESULT + OUTPUT_VARIABLE GIT_COMMIT) + if (GIT_RESULT) + set (GIT_COMMIT "UNKNOWN") + endif() +set(ZMAP_VERSION "Development Build. Commit ${GIT_COMMIT}") + +configure_file("${ORIG_SRC_DIR}/src/topt.ggo.in" "${CMAKE_BINARY_DIR}/topt.ggo" @ONLY) +configure_file("${ORIG_SRC_DIR}/src/zbopt.ggo.in" "${CMAKE_BINARY_DIR}/zbopt.ggo" @ONLY) +configure_file("${ORIG_SRC_DIR}/src/zitopt.ggo.in" "${CMAKE_BINARY_DIR}/zitopt.ggo" @ONLY) +configure_file("${ORIG_SRC_DIR}/src/zopt.ggo.in" "${CMAKE_BINARY_DIR}/zopt.ggo" @ONLY) +configure_file("${ORIG_SRC_DIR}/src/ztopt.ggo.in" "${CMAKE_BINARY_DIR}/ztopt.ggo" @ONLY) diff --git a/src/aesrand.c b/src/aesrand.c new file mode 100644 index 0000000..cf957d5 --- /dev/null +++ b/src/aesrand.c @@ -0,0 +1,71 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include + +#include "../lib/logger.h" +#include "../lib/rijndael-alg-fst.h" +#include "../lib/random.h" +#include "../lib/xalloc.h" + +#include "aesrand.h" + +#define AES_ROUNDS 10 +#define AES_BLOCK_WORDS 4 +#define AES_KEY_BYTES 16 +#define AES_KEY_BITS (AES_KEY_BYTES * 8) +#define OUTPUT_BYTES 16 + +struct aesrand { + uint32_t input[AES_BLOCK_WORDS]; + uint32_t sched[(AES_ROUNDS + 1) * 4]; + uint8_t output[OUTPUT_BYTES]; +}; + +static aesrand_t *_aesrand_init(uint8_t *key) +{ + aesrand_t *aes = xmalloc(sizeof(aesrand_t)); + memset(&aes->input, 0, AES_BLOCK_WORDS * 4); + if (rijndaelKeySetupEnc(aes->sched, key, AES_KEY_BITS) != AES_ROUNDS) { + log_fatal("aesrand", "could not initialize AES key"); + } + memset(aes->output, 0, OUTPUT_BYTES); + return aes; +} + +aesrand_t *aesrand_init_from_seed(uint64_t seed) +{ + uint8_t key[AES_KEY_BYTES]; + memset(key, 0, AES_KEY_BYTES); + for (uint8_t i = 0; i < sizeof(seed); ++i) { + key[i] = (uint8_t)((seed >> 8 * i) & 0xFF); + } + return _aesrand_init(key); +} + +aesrand_t *aesrand_init_from_random(void) +{ + uint8_t key[AES_KEY_BYTES]; + if (!random_bytes(key, AES_KEY_BYTES)) { + log_fatal("aesrand", "Couldn't get random bytes"); + } + return _aesrand_init(key); +} + +uint64_t aesrand_getword(aesrand_t *aes) +{ + memcpy(aes->input, aes->output, sizeof(aes->input)); + rijndaelEncrypt(aes->sched, AES_ROUNDS, (uint8_t *)aes->input, + aes->output); + uint64_t retval; + memcpy(&retval, aes->output, sizeof(retval)); + return retval; +} diff --git a/src/aesrand.h b/src/aesrand.h new file mode 100644 index 0000000..2754d7f --- /dev/null +++ b/src/aesrand.h @@ -0,0 +1,24 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include + +#ifndef AESRAND_H +#define AESRAND_H + +typedef struct aesrand aesrand_t; + +aesrand_t *aesrand_init_from_random(void); + +aesrand_t *aesrand_init_from_seed(uint64_t); + +uint64_t aesrand_getword(aesrand_t *aes); + +aesrand_t *aesrand_free(aesrand_t *aes); + +#endif diff --git a/src/constants.h b/src/constants.h new file mode 100644 index 0000000..2fe8db7 --- /dev/null +++ b/src/constants.h @@ -0,0 +1,16 @@ +/* + * ZMap Copyright 2023 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_CONSTANTS_H +#define ZMAP_CONSTANTS_H + +#ifndef ZMAP_DEFAULT_BLOCKLIST +#define ZMAP_DEFAULT_BLOCKLIST "/etc/zmap/blocklist.conf" +#endif + +#endif /* ZMAP_CONSTANTS_H */ diff --git a/src/cyclic.c b/src/cyclic.c new file mode 100644 index 0000000..cf1a66e --- /dev/null +++ b/src/cyclic.c @@ -0,0 +1,191 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * cyclic provides an inexpensive approach to iterating over the IPv4 address + * space in a random(-ish) manner such that we connect to every host once in + * a scan execution without having to keep track of the IPs that have been + * scanned or need to be scanned and such that each scan has a different + * ordering. We accomplish this by utilizing a cyclic multiplicative group + * of integers modulo a prime and generating a new primitive root (generator) + * for each scan. + * + * We know that 3 is a generator of (Z mod 2^32 + 15 - {0}, *) + * and that we have coverage over the entire address space because 2**32 + 15 + * is prime and ||(Z mod PRIME - {0}, *)|| == PRIME - 1. Therefore, we + * just need to find a new generator (primitive root) of the cyclic group for + * each scan that we perform. + * + * Because generators map to generators over an isomorphism, we can efficiently + * find random primitive roots of our mult. group by finding random generators + * of the group (Zp-1, +) which is isomorphic to (Zp*, *). Specifically the + * generators of (Zp-1, +) are { s | (s, p-1) == 1 } which implies that + * the generators of (Zp*, *) are { d^s | (s, p-1) == 1 }. where d is a known + * generator of the multiplicative group. We efficiently find + * generators of the additive group by precalculating the psub1_f of + * p - 1 and randomly checking random numbers against the psub1_f until + * we find one that is coprime and map it into Zp*. Because + * totient(totient(p)) ~= 10^9, this should take relatively few + * iterations to find a new generator. + */ + +#include "cyclic.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" + +// We will pick the first cyclic group from this list that is +// larger than the number of IPs in our allowlist. E.g. for an +// entire Internet scan, this would be cyclic32 +// Note: this list should remain ordered by size (primes) ascending. + +static cyclic_group_t groups[] = { + {// 2^8 + 1 + .prime = 257, + .known_primroot = 3, + .prime_factors = {2}, + .num_prime_factors = 1}, + {// 2^16 + 1 + .prime = 65537, + .known_primroot = 3, + .prime_factors = {2}, + .num_prime_factors = 1}, + {// 2^24 + 43 + .prime = 16777259, + .known_primroot = 2, + .prime_factors = {2, 23, 103, 3541}, + .num_prime_factors = 4}, + {// 2^28 + 3 + .prime = 268435459, + .known_primroot = 2, + .prime_factors = {2, 3, 19, 87211}, + .num_prime_factors = 4}, + {// 2^32 + 15 + .prime = 4294967311, + .known_primroot = 3, + .prime_factors = {2, 3, 5, 131, 364289}, + .num_prime_factors = 5}, + {// 2^33 + 17 + .prime = 8589934609, + .known_primroot = 19, + .prime_factors = {2, 3, 59, 3033169}, + .num_prime_factors = 4}, + {// 2^34 + 25 + .prime = 17179869209, + .known_primroot = 3, + .prime_factors = {2, 83, 1277, 20261}, + .num_prime_factors = 4}, + {// 2^36 + 31 + .prime = 68719476767, + .known_primroot = 5, + .prime_factors = {2, 163, 883, 238727}, + .num_prime_factors = 4}, + {// 2^40 + 15 + .prime = 1099511627791, + .known_primroot = 3, + .prime_factors = {2, 3, 5, 36650387593}, + .num_prime_factors = 4}, + {// 2^44 + 7 + .prime = 17592186044423, + .known_primroot = 5, + .prime_factors = {2, 11, 53, 97, 155542661}, + .num_prime_factors = 5}, + {// 2^48 + 23 + .prime = 281474976710677, + .known_primroot = 6, + .prime_factors = {2, 3, 7, 1361, 2462081249}, + .num_prime_factors = 5}, +}; + +// Return a (random) number coprime with (p - 1) of the group, +// which is a generator of the additive group mod (p - 1) +static uint32_t find_primroot(const cyclic_group_t *group, aesrand_t *aes) +{ + uint32_t candidate = + (uint32_t)((aesrand_getword(aes) & 0xFFFFFFFF) % group->prime); + uint64_t retv = 0; + + // The maximum primitive root we can return needs to be small enough such + // that there is no overflow when multiplied by any element in the largest + // group in ZMap, which currently has p = 2^{32} + 15. + const uint64_t max_root = (UINT64_C(1) << 22); + + // Repeatedly find a generator until we hit one that is small enough. For + // the largest group, we have a very low probability of ever executing this + // loop more than once, and for small groups it will only execute once. + do { + candidate += 1; + + // Only one of these mods will ever have an effect. + candidate %= group->prime; + candidate %= max_root; + + if (candidate == 0) { + continue; + } + + mpz_t prime; + mpz_init_set_ui(prime, group->prime); + int ok = 1; + for (size_t i = 0; i < group->num_prime_factors && ok; ++i) { + const uint64_t q = group->prime_factors[i]; + const uint64_t k = (group->prime - 1) / q; + mpz_t base, power, res; + mpz_init_set_ui(base, candidate); + mpz_init_set_ui(power, k); + mpz_init(res); + mpz_powm(res, base, power, prime); + uint64_t res_ui = mpz_get_ui(res); + if (res_ui == 1) { + ok = 0; + } + mpz_clear(base); + mpz_clear(power); + mpz_clear(res); + } + if (ok) { + retv = candidate; + break; + } + } while (1); + log_debug("zmap", "Isomorphism: %llu", retv); + return retv; +} + +const cyclic_group_t *get_group(uint64_t min_size) +{ + for (unsigned i = 0; i < sizeof(groups); ++i) { + if (groups[i].prime > min_size) { + return &groups[i]; + } + } + // Should not reach, final group should always be larger than 2^48 + // which is max based on 2**32 IPs and 2**16 ports + assert(0); +} + +cycle_t make_cycle(const cyclic_group_t *group, aesrand_t *aes) +{ + cycle_t cycle; + cycle.group = group; + cycle.generator = find_primroot(group, aes); + cycle.offset = (uint32_t)(aesrand_getword(aes) & 0xFFFFFFFF); + cycle.offset %= group->prime; + cycle.order = group->prime - 1; + return cycle; +} diff --git a/src/cyclic.h b/src/cyclic.h new file mode 100644 index 0000000..c587df4 --- /dev/null +++ b/src/cyclic.h @@ -0,0 +1,45 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef CYCLIC_H +#define CYCLIC_H + +#include +#include + +#include "aesrand.h" + +// Represents a multiplicative cyclic group (Z/pZ)* +typedef struct cyclic_group { + uint64_t prime; // p + uint64_t known_primroot; // Known primitive root of (Z/pZ)* + size_t num_prime_factors; // Length of num_prime_factors + uint64_t prime_factors[10]; // Unique prime factors of (p-1) +} cyclic_group_t; + +// Represents a cycle in a group +typedef struct cycle { + const cyclic_group_t *group; + uint64_t generator; + uint64_t order; + uint32_t offset; +} cycle_t; + +// Get a cyclic_group_t of at least min_size. +// Pointer into static data, do not free(). +const cyclic_group_t *get_group(uint64_t min_size); + +// Generate cycle (find generator and inverse) +cycle_t make_cycle(const cyclic_group_t *group, aesrand_t *aes); + +// Perform the isomorphism from (Z/pZ)+ to (Z/pZ)* +// Given known primitive root of (Z/pZ)* n, with x in (Z/pZ)+, do: +// f(x) = n^x mod p +// + +#endif diff --git a/src/expression.c b/src/expression.c new file mode 100644 index 0000000..3fc1107 --- /dev/null +++ b/src/expression.c @@ -0,0 +1,170 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "expression.h" +#include "fieldset.h" + +#include "../lib/xalloc.h" + +/* Static helper functions */ + +static node_t *alloc_node(void); +static int eval_gt_node(node_t *node, fieldset_t *fields); +static int eval_lt_node(node_t *node, fieldset_t *fields); +static int eval_eq_node(node_t *node, fieldset_t *fields); +static int eval_lt_eq_node(node_t *node, fieldset_t *fields); +static int eval_gt_eq_node(node_t *node, fieldset_t *fields); + +static node_t *alloc_node(void) +{ + node_t *node = xmalloc(sizeof(node_t)); + return node; +} + +static int eval_gt_node(node_t *node, fieldset_t *fields) +{ + int index = node->left_child->value.field.index; + uint64_t expected = node->right_child->value.int_literal; + uint64_t actual = fs_get_uint64_by_index(fields, index); + return (actual > expected); +} + +static int eval_lt_node(node_t *node, fieldset_t *fields) +{ + int index = node->left_child->value.field.index; + uint64_t expected = node->right_child->value.int_literal; + uint64_t actual = fs_get_uint64_by_index(fields, index); + return (actual < expected); +} + +static int eval_eq_node(node_t *node, fieldset_t *fields) +{ + node_t *literal = node->right_child; + int index = node->left_child->value.field.index; + char *expected, *actual; + switch (literal->type) { + case STRING: + expected = literal->value.string_literal; + actual = fs_get_string_by_index(fields, index); + return (strcmp(expected, actual) == 0); + break; + case INT: + return (fs_get_uint64_by_index(fields, index) == + literal->value.int_literal); + break; + default: + printf("wat\n"); + break; + } + return 0; +} + +static int eval_lt_eq_node(node_t *node, fieldset_t *fields) +{ + return !(eval_gt_node(node, fields)); +} + +static int eval_gt_eq_node(node_t *node, fieldset_t *fields) +{ + return !(eval_lt_node(node, fields)); +} + +/* Exposed functions */ + +node_t *make_op_node(enum operation op) +{ + node_t *node = alloc_node(); + node->type = OP; + node->value.op = op; + return node; +} + +node_t *make_field_node(char *fieldname) +{ + node_t *node = alloc_node(); + node->type = FIELD; + node->value.field.fieldname = fieldname; + return node; +} + +node_t *make_string_node(char *literal) +{ + node_t *node = alloc_node(); + node->type = STRING; + node->value.string_literal = literal; + return node; +} + +node_t *make_int_node(int literal) +{ + node_t *node = alloc_node(); + node->type = INT; + node->value.int_literal = literal; + return node; +} + +int evaluate_expression(node_t *root, fieldset_t *fields) +{ + if (!root) + return 1; + switch (root->type) { /* XXX Not sure if runs */ + case FIELD: + case STRING: + case INT: + return 1; + case OP: + break; + } + switch (root->value.op) { + case GT: + return eval_gt_node(root, fields); + case LT: + return eval_lt_node(root, fields); + case EQ: + return eval_eq_node(root, fields); + case NEQ: + return (!eval_eq_node(root, fields)); + case LT_EQ: + return eval_lt_eq_node(root, fields); + case GT_EQ: + return eval_gt_eq_node(root, fields); + case AND: + return (evaluate_expression(root->left_child, fields) && + evaluate_expression(root->right_child, fields)); + case OR: + return (evaluate_expression(root->left_child, fields) || + evaluate_expression(root->right_child, fields)); + } + return 0; +} + +void print_expression(node_t *root) +{ + if (!root) + return; + printf("%s", "( "); + print_expression(root->left_child); + switch (root->type) { + case OP: + printf(" %i ", root->value.op); + break; + case FIELD: + printf(" (%s", root->value.field.fieldname); + break; + case STRING: + printf("%s) ", root->value.string_literal); + break; + case INT: + printf(" %llu) ", (long long unsigned)root->value.int_literal); + break; + default: + break; + } + print_expression(root->right_child); + printf("%s", " )"); +} diff --git a/src/expression.h b/src/expression.h new file mode 100644 index 0000000..d6a5f32 --- /dev/null +++ b/src/expression.h @@ -0,0 +1,54 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_TREE_H +#define ZMAP_TREE_H + +#include "fieldset.h" + +#include +#include +#include +#include + +enum operation { GT, LT, EQ, NEQ, AND, OR, LT_EQ, GT_EQ }; + +enum node_type { OP, FIELD, STRING, INT }; + +struct field_id { + int index; + char *fieldname; +}; + +union node_value { + struct field_id field; + char *string_literal; + uint64_t int_literal; + enum operation op; +}; + +typedef struct node_st { + struct node_st *left_child; + struct node_st *right_child; + enum node_type type; + union node_value value; +} node_t; + +node_t *make_op_node(enum operation op); + +node_t *make_field_node(char *fieldname); + +node_t *make_string_node(char *literal); + +node_t *make_int_node(int literal); + +int evaluate_expression(node_t *root, fieldset_t *fields); + +void print_expression(node_t *root); + +#endif /* ZMAP_TREE_H */ diff --git a/src/fieldset.c b/src/fieldset.c new file mode 100644 index 0000000..bd36c44 --- /dev/null +++ b/src/fieldset.c @@ -0,0 +1,410 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "fieldset.h" + +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wundef" +#include +#pragma GCC diagnostic pop + +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len) +{ + if (fds->len + len > MAX_FIELDS) { + log_fatal("fieldset", "out of room in field def set"); + } + fielddef_t *open = &(fds->fielddefs[fds->len]); + memcpy(open, fs, len * sizeof(fielddef_t)); + fds->len += len; +} + +fieldset_t *fs_new_fieldset(fielddefset_t *fds) +{ + fieldset_t *f = xcalloc(1, sizeof(fieldset_t)); + f->len = 0; + f->type = FS_FIELDSET; + f->fds = fds; + return f; +} + +fieldset_t *fs_new_repeated_field(int type, int free_) +{ + fieldset_t *f = xcalloc(1, sizeof(fieldset_t)); + f->len = 0; + f->type = FS_REPEATED; + f->inner_type = type; + f->free_ = free_; + return f; +} + +fieldset_t *fs_new_repeated_uint64(void) +{ + return fs_new_repeated_field(FS_UINT64, 0); +} + +fieldset_t *fs_new_repeated_bool(void) +{ + return fs_new_repeated_field(FS_BOOL, 0); +} + +fieldset_t *fs_new_repeated_string(int free_) +{ + return fs_new_repeated_field(FS_STRING, free_); +} + +fieldset_t *fs_new_repeated_binary(int free_) +{ + return fs_new_repeated_field(FS_BINARY, free_); +} + +fieldset_t *fs_new_repeated_fieldset(void) +{ + return fs_new_repeated_field(FS_FIELDSET, 0); +} + +static inline void fs_add_word(fieldset_t *fs, const char *name, int type, + int free_, size_t len, field_val_t value) +{ + if (fs->len + 1 >= MAX_FIELDS) { + log_fatal("fieldset", "out of room in fieldset"); + } + if (fs->type == FS_REPEATED && fs->inner_type != type) { + log_fatal( + "fieldset", + "object added to repeated field does not match type of repeated field."); + } + field_t *f = &(fs->fields[fs->len]); + // if we have a fieldset definition, then we can validate that the name + // of the field is as expected + if (fs->fds && strcmp(fs->fds->fielddefs[fs->len].name, name)) { + log_fatal("fieldset", + "added field (%s) is not next expected field (%s).", + name, fs->fds->fielddefs[fs->len].name); + } + + fs->len++; + f->type = type; + f->name = name; + f->len = len; + f->value = value; + f->free_ = free_; +} + +static void fs_modify_word(fieldset_t *fs, const char *name, int type, + int free_, size_t len, field_val_t value) +{ + for (int i = 0; i < fs->len; i++) { + if (!strcmp(fs->fields[i].name, name)) { + if (fs->fields[i].free_) { + free(fs->fields[i].value.ptr); + fs->fields[i].value.ptr = NULL; + } + fs->fields[i].type = type; + fs->fields[i].free_ = free_; + fs->fields[i].len = len; + fs->fields[i].value = value; + return; + } + } + // TODO(ZD): We need to test, but this is really unsafe to just add because it + // will all but guarantee that it's in the wrong place + //fs_add_word(fs, name, type, free_, len, value); + log_fatal("fs", "attempting to modify non-existent field"); +} + +static char *sanitize_utf8(const char *buf) +{ + const char *ptr = buf; + + // Count how many errors we encounter + uint32_t i = 0; + // Upper bounds to ensure termination even if u8_check is unsafe + while (i < strlen(buf) && ptr < buf + strlen(buf)) { + ptr = (char *)u8_check((uint8_t *)ptr, strlen(ptr)); + if (ptr == NULL) { + break; + } + + assert(ptr >= buf); + assert(ptr < buf + strlen(buf)); + + ptr++; + i++; + } + + // i is the total number of errors. We need 2 extra bytes for each rune + char *safe_buf = xmalloc(strlen(buf) + i * 2 + 1); + char *safe_ptr = NULL; + memcpy(safe_buf, buf, strlen(buf)); + + // Fix exactly i errors + for (uint32_t j = 0; j < i; j++) { + // Always operate on the working buffer + safe_ptr = + (char *)u8_check((uint8_t *)safe_buf, strlen(safe_buf)); + + // This implies we had less errors than we should. + // This is temporary debug code. + if (safe_ptr == NULL) { + log_warn( + "fieldset", + "UTF8 Sanitization issue. %u errors, fell through iter %u. Orig: %s new: %s", + i, j, buf, safe_buf); + i = j; + break; + } + + // XXX Uncomment when we remove above log_warn. + // assert(safe_ptr != NULL); + assert(safe_ptr >= safe_buf); + assert(safe_ptr < safe_buf + strlen(safe_buf)); + + // Shift the rest of the string by 2 bytes + if (strlen(safe_ptr) > 1) { + memcpy(safe_ptr + 3, safe_ptr + 1, + strlen(safe_ptr + 1)); + } + + // UTF8 replacement rune + safe_ptr[0] = (char)0xef; + safe_ptr[1] = (char)0xbf; + safe_ptr[2] = (char)0xbd; + } + + // We now have a valid utf8 string + assert(u8_check((uint8_t *)safe_buf, strlen(safe_buf)) == NULL); + // We should be null terminated + assert(safe_buf[strlen(buf) + i * 2] == '\0'); + // We should be the right length + assert(strlen(safe_buf) == (strlen(buf) + i * 2)); + + return safe_buf; +} + +void fs_add_null(fieldset_t *fs, const char *name) +{ + field_val_t val = {.ptr = NULL}; + fs_add_word(fs, name, FS_NULL, 0, 0, val); +} + +void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_) +{ + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_STRING, free_, strlen(value), val); +} + +void fs_add_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_) +{ + if (u8_check((uint8_t *)value, strlen(value)) == NULL) { + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_STRING, free_, strlen(value), val); + } else { + char *safe_value = sanitize_utf8(value); + + if (free_) { + free(value); + } + + field_val_t val = {.ptr = safe_value}; + fs_add_word(fs, name, FS_STRING, 1, strlen(safe_value), val); + } +} + +void fs_chkadd_string(fieldset_t *fs, const char *name, char *value, int free_) +{ + if (value) { + fs_add_string(fs, name, value, free_); + } else { + fs_add_null(fs, name); + } +} + +void fs_chkadd_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_) +{ + if (value) { + fs_add_unsafe_string(fs, name, value, free_); + } else { + fs_add_null(fs, name); + } +} + +void fs_add_constchar(fieldset_t *fs, const char *name, const char *value) +{ + field_val_t val = {.ptr = (char *)value}; + fs_add_word(fs, name, FS_STRING, 0, strlen(value), val); +} + +void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value) +{ + field_val_t val = {.num = value}; + fs_add_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), val); +} + +void fs_add_bool(fieldset_t *fs, const char *name, int value) +{ + field_val_t val = {.num = value}; + fs_add_word(fs, name, FS_BOOL, 0, sizeof(int), val); +} + +void fs_add_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_) +{ + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_BINARY, free_, len, val); +} + +void fs_add_fieldset(fieldset_t *fs, const char *name, fieldset_t *child) +{ + field_val_t val = {.ptr = child}; + fs_add_word(fs, name, FS_FIELDSET, 1, sizeof(void *), val); +} + +void fs_add_repeated(fieldset_t *fs, const char *name, fieldset_t *child) +{ + field_val_t val = {.ptr = child}; + fs_add_word(fs, name, FS_REPEATED, 1, sizeof(void *), val); +} + +// Modify +void fs_modify_null(fieldset_t *fs, const char *name) +{ + field_val_t val = {.ptr = NULL}; + fs_modify_word(fs, name, FS_NULL, 0, 0, val); +} + +void fs_modify_string(fieldset_t *fs, const char *name, char *value, int free_) +{ + field_val_t val = {.ptr = value}; + fs_modify_word(fs, name, FS_STRING, free_, strlen(value), val); +} + +void fs_modify_constchar(fieldset_t *fs, const char *name, const char *value) +{ + field_val_t val = {.ptr = (char *)value}; + fs_modify_word(fs, name, FS_STRING, 0, strlen(value), val); +} + +void fs_modify_uint64(fieldset_t *fs, const char *name, uint64_t value) +{ + field_val_t val = {.num = value}; + fs_modify_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), val); +} + +void fs_modify_bool(fieldset_t *fs, const char *name, int value) +{ + field_val_t val = {.num = value}; + fs_modify_word(fs, name, FS_BOOL, 0, sizeof(int), val); +} + +void fs_modify_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_) +{ + field_val_t val = {.ptr = value}; + fs_modify_word(fs, name, FS_BINARY, free_, len, val); +} + +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index) +{ + return (uint64_t)fs->fields[index].value.num; +} + +char *fs_get_string_by_index(fieldset_t *fs, int index) +{ + return (char *)fs->fields[index].value.ptr; +} + +int fds_get_index_by_name(fielddefset_t *fds, const char *name) +{ + for (int i = 0; i < fds->len; i++) { + if (!strcmp(fds->fielddefs[i].name, name)) { + return i; + } + } + return -1; +} + +void field_free(field_t *f) +{ + if (f->type == FS_FIELDSET || f->type == FS_REPEATED) { + fs_free((fieldset_t *)f->value.ptr); + } else if (f->free_) { + free(f->value.ptr); + } +} + +void fs_free(fieldset_t *fs) +{ + if (!fs) { + return; + } + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + field_free(f); + } + free(fs); +} + +void fs_generate_fieldset_translation(translation_t *t, fielddefset_t *avail, + const char **req, int reqlen) +{ + memset(t, 0, sizeof(translation_t)); + if (!t) { + log_fatal("fieldset", + "unable to allocate memory for translation"); + } + for (int i = 0; i < reqlen; i++) { + int l = fds_get_index_by_name(avail, req[i]); + if (l < 0) { + log_fatal("fieldset", + "specified field (%s) not " + "available in selected " + "probe module.", + req[i]); + } + t->translation[t->len++] = l; + } +} + +void fs_generate_full_fieldset_translation(translation_t *t, + fielddefset_t *avail) +{ + memset(t, 0, sizeof(translation_t)); + if (!t) { + log_fatal("fieldset", + "unable to allocate memory for translation"); + } + t->len = avail->len; + for (int i = 0; i < avail->len; i++) { + t->translation[i] = i; + } +} + +fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t) +{ + fieldset_t *retv = fs_new_fieldset(NULL); + if (!retv) { + log_fatal("fieldset", + "unable to allocate space for translated field set"); + } + for (int i = 0; i < t->len; i++) { + int o = t->translation[i]; + memcpy(&(retv->fields[i]), &(fs->fields[o]), sizeof(field_t)); + } + retv->len = t->len; + return retv; +} diff --git a/src/fieldset.h b/src/fieldset.h new file mode 100644 index 0000000..97ddc2d --- /dev/null +++ b/src/fieldset.h @@ -0,0 +1,147 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include "types.h" + +#ifndef FIELDSET_H +#define FIELDSET_H + +// maximum number of records that can be stored in a fieldset +#define MAX_FIELDS 128 +#define MAX_LIST_LENGTH 255 + +// types of data that can be stored in a field +#define FS_RESERVED 0 +#define FS_STRING 1 +#define FS_UINT64 2 +#define FS_BINARY 3 +#define FS_NULL 4 +#define FS_BOOL 7 +// recursive support +#define FS_FIELDSET 5 +#define FS_REPEATED 6 + +// definition of a field that's provided by a probe module +// these are used so that users can ask at the command-line +// what fields are available for consumption +typedef struct field_def { + const char *name; + const char *type; + const char *desc; +} fielddef_t; + +typedef struct fielddef_set { + fielddef_t fielddefs[MAX_FIELDS]; + int len; +} fielddefset_t; + +typedef union field_val { + void *ptr; + uint64_t num; +} field_val_t; + +// the internal field type used by fieldset +typedef struct field { + const char *name; + int type; + int free_; + size_t len; + field_val_t value; +} field_t; + +// data structure that is populated by the probe module +// and translated into the data structure that's passed +// to the output module +typedef struct fieldset { + int len; + field_t fields[MAX_FIELDS]; + fielddefset_t *fds; + // only used for repeated. + int inner_type; // type of repeated element. e.g., FS_STRING + int type; // REPEATED or FIELDSET + int free_; // should elements be freed +} fieldset_t; + +// we pass a different fieldset to an output module than +// the probe module generates for us because a user may +// only want certain fields and will expect them in a certain +// order. We generate a translated fieldset that contains +// only the fields we want to export to the output module. +// a translation specifies how to efficiently convert the fs +// povided by the probe module to the fs for the output module. +typedef struct translation { + int len; + int translation[MAX_FIELDS]; +} translation_t; + +fieldset_t *fs_new_fieldset(fielddefset_t *); + +fieldset_t *fs_new_repeated_field(int type, int free_); +fieldset_t *fs_new_repeated_uint64(void); +fieldset_t *fs_new_repeated_bool(void); +fieldset_t *fs_new_repeated_string(int free_); +fieldset_t *fs_new_repeated_binary(int free_); +fieldset_t *fs_new_repeated_fieldset(void); + +char *fs_get_string_by_index(fieldset_t *fs, int index); + +int fds_get_index_by_name(fielddefset_t *fds, const char *name); + +void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len); + +void fs_add_null(fieldset_t *fs, const char *name); + +void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value); + +void fs_add_bool(fieldset_t *fs, const char *name, int value); + +void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_add_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_); + +void fs_chkadd_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_chkadd_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_); + +void fs_add_constchar(fieldset_t *fs, const char *name, const char *value); + +void fs_add_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_); + +void fs_add_fieldset(fieldset_t *fs, const char *name, fieldset_t *child); +void fs_add_repeated(fieldset_t *fs, const char *name, fieldset_t *child); + +// Modify +void fs_modify_null(fieldset_t *fs, const char *name); + +void fs_modify_uint64(fieldset_t *fs, const char *name, uint64_t value); + +void fs_modify_bool(fieldset_t *fs, const char *name, int value); + +void fs_modify_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_modify_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_); + +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index); + +void fs_free(fieldset_t *fs); + +void fs_generate_fieldset_translation(translation_t *t, fielddefset_t *avail, + const char **req, int reqlen); + +fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t); + +void fs_generate_full_fieldset_translation(translation_t *t, + fielddefset_t *avail); + +#endif // FIELDSET_H diff --git a/src/filter.c b/src/filter.c new file mode 100644 index 0000000..27e69c1 --- /dev/null +++ b/src/filter.c @@ -0,0 +1,115 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "filter.h" +#include "state.h" +#include "lexer.h" +#include "parser.h" +#include "expression.h" +#include "../lib/logger.h" + +#include + +extern int yyparse(void); + +node_t *zfilter; + +static int validate_node(node_t *node, fielddefset_t *fields) +{ + int index, found = 0; + if (node->type == OP) { + // These end up getting validated later + if (node->value.op == AND || node->value.op == OR) { + return 1; + } + // Comparison node (=, >, <, etc.) + // Validate that the field (left child) exists in the fieldset + for (index = 0; index < fields->len; index++) { + if (fields->fielddefs[index].name) { + if (strcmp(fields->fielddefs[index].name, + node->left_child->value.field + .fieldname) == 0) { + node->left_child->value.field.index = + index; + found = 1; + break; + } + } + } + if (!found) { + fprintf(stderr, "Field '%s' does not exist\n", + node->left_child->value.field.fieldname); + return 0; + } + // Fieldname is fine, match the type. + switch (node->right_child->type) { + case STRING: + if (strcmp(fields->fielddefs[index].type, "string") == + 0) { + return 1; + } else { + fprintf(stderr, + "Field '%s' is not of type 'string'\n", + fields->fielddefs[index].name); + return 0; + } + case INT: + if (strcmp(fields->fielddefs[index].type, "int") == 0 || + strcmp(fields->fielddefs[index].type, "bool") == + 0) { + return 1; + } else { + fprintf(stderr, + "Field '%s' is not of type 'int'\n", + fields->fielddefs[index].name); + return 0; + } + default: + return 0; + } + } else { + // All non-op nodes are valid + return 1; + } + // Didn't validate + return 0; +} + +int parse_filter_string(char *filter) +{ + YY_BUFFER_STATE buffer_state = yy_scan_string(filter); + int status = yyparse(); + yy_delete_buffer(buffer_state); + if (status) { + // Error + log_error("zmap", "Unable to parse filter string: '%s'", + filter); + return 0; + } + zconf.filter.expression = zfilter; + return 1; +} + +/* + * 0 Valid + * -1 Invalid Field Name + * -2 Type Mismatch + */ +int validate_filter(node_t *root, fielddefset_t *fields) +{ + int valid; + if (!root) { + return 1; + } + valid = validate_node(root, fields); + if (!valid) { + return 0; + } + return (validate_filter(root->left_child, fields) && + validate_filter(root->right_child, fields)); +} diff --git a/src/filter.h b/src/filter.h new file mode 100644 index 0000000..6e23cc5 --- /dev/null +++ b/src/filter.h @@ -0,0 +1,23 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_FILTER_H +#define ZMAP_FILTER_H + +#include "expression.h" +#include "fieldset.h" + +struct output_filter { + node_t *expression; +}; + +int parse_filter_string(char *filter); + +int validate_filter(node_t *root, fielddefset_t *fields); + +#endif /* ZMAP_FILTER_H */ diff --git a/src/get_gateway-bsd.h b/src/get_gateway-bsd.h new file mode 100644 index 0000000..0e8d174 --- /dev/null +++ b/src/get_gateway-bsd.h @@ -0,0 +1,224 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_GET_GATEWAY_BSD_H +#define ZMAP_GET_GATEWAY_BSD_H + +#ifdef ZMAP_GET_GATEWAY_LINUX_H +#error "Don't include both get_gateway-bsd.h and get_gateway-linux.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ROUNDUP(a) ((a) > 0 ? (1 + (((a)-1) | (sizeof(int) - 1))) : sizeof(int)) +#define UNUSED __attribute__((unused)) + +int get_hw_addr(struct in_addr *gw_ip, UNUSED char *iface, + unsigned char *hw_mac) +{ + int mib[6]; + mib[0] = CTL_NET; + mib[1] = PF_ROUTE; + mib[2] = 0; + mib[3] = AF_INET; + mib[4] = NET_RT_FLAGS; + mib[5] = RTF_LLINFO; + size_t bufsz = 0; + if (sysctl(mib, 6, NULL, &bufsz, NULL, 0) == -1) { + log_debug("get_hw_addr", "sysctl getting buffer size: %d %s", errno, strerror(errno)); + return EXIT_FAILURE; + } + uint8_t *buf = (uint8_t *)malloc(bufsz); + assert(buf); + if (sysctl(mib, 6, buf, &bufsz, NULL, 0) == -1) { + log_debug("get_hw_addr", "sysctl getting buffer data: %d %s", errno, strerror(errno)); + free(buf); + return EXIT_FAILURE; + } + + int result = EXIT_FAILURE; + uint8_t *bufend = buf + bufsz; + size_t min_msglen = sizeof(struct rt_msghdr) + sizeof(struct sockaddr_inarp) + sizeof(struct sockaddr_dl); + struct rt_msghdr *rtm = (struct rt_msghdr *)buf; + for (uint8_t *p = buf; p < bufend; p += rtm->rtm_msglen) { + rtm = (struct rt_msghdr *)p; + if ((p + sizeof(struct rt_msghdr) > bufend) || + (p + rtm->rtm_msglen > bufend) || + (rtm->rtm_msglen < min_msglen)) { + break; + } + struct sockaddr_inarp *sin = (struct sockaddr_inarp *)(rtm + 1); + struct sockaddr_dl *sdl = (struct sockaddr_dl *)(sin + 1); + assert(sin->sin_family == AF_INET); + if (sin->sin_addr.s_addr != gw_ip->s_addr) { + continue; + } + assert(sdl->sdl_family == AF_LINK); + memcpy(hw_mac, LLADDR(sdl), ETHER_ADDR_LEN); + result = EXIT_SUCCESS; + } + + free(buf); + return result; +} + +int get_iface_ip(char *iface, struct in_addr *ip) +{ + assert(iface); + struct ifaddrs *ifaddr, *ifa; + if (getifaddrs(&ifaddr)) { + log_fatal( + "get-iface-ip", + "ZMap is unable able to retrieve a list of available network " + "interfaces: %s. You can manually specify the network interface " + "to use with the \"-i\" flag.", + strerror(errno)); + } + for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == NULL || + ifa->ifa_addr->sa_family != AF_INET) { + continue; + } + if (!strcmp(iface, ifa->ifa_name)) { + struct sockaddr_in *sin = + (struct sockaddr_in *)ifa->ifa_addr; + ip->s_addr = sin->sin_addr.s_addr; + log_debug("get-iface-ip", "IP address found for %s: %s", + iface, inet_ntoa(*ip)); + freeifaddrs(ifaddr); + return EXIT_SUCCESS; + } + } + log_fatal("get-iface-ip", + "The specified network interface (\"%s\") does not" + " exist or does not have an assigned IPv4 address.", + iface); + return EXIT_FAILURE; +} + +int get_iface_hw_addr(char *iface, unsigned char *hw_mac) +{ + struct ifaddrs *ifa; + if (getifaddrs(&ifa) == -1) { + log_debug("get_iface_hw_addr", "getifaddrs(): %d %s", errno, strerror(errno)); + return EXIT_FAILURE; + } + int result = EXIT_FAILURE; + for (struct ifaddrs *p = ifa; p; p = p->ifa_next) { + if (strcmp(p->ifa_name, iface) == 0 && + p->ifa_addr != NULL && p->ifa_addr->sa_family == AF_LINK) { + struct sockaddr_dl *sdl = (struct sockaddr_dl *)p->ifa_addr; + memcpy(hw_mac, LLADDR(sdl), ETHER_ADDR_LEN); + result = EXIT_SUCCESS; + break; + } + } + freeifaddrs(ifa); + return result; +} + +int _get_default_gw(struct in_addr *gw, char **iface) +{ + char buf[4096]; + struct rt_msghdr *rtm = (struct rt_msghdr *)&buf; + memset(rtm, 0, sizeof(buf)); + int seq = 0x00FF; + rtm->rtm_msglen = sizeof(buf); + rtm->rtm_type = RTM_GET; + rtm->rtm_flags = RTF_GATEWAY; + rtm->rtm_version = RTM_VERSION; + rtm->rtm_seq = seq; + rtm->rtm_addrs = RTA_DST | RTA_IFP; + rtm->rtm_pid = getpid(); + + int fd = socket(PF_ROUTE, SOCK_RAW, 0); + assert(fd > 0); + if (!write(fd, (char *)rtm, sizeof(buf))) { + log_fatal( + "get-gateway", + "Unable to send request to retrieve default" + "gateway MAC address. You will need to manually specify your " + "gateway MAC with the \"-G\" or \"--gateway-mac\" flag."); + } + size_t len; + while (rtm->rtm_type == RTM_GET && + (len = read(fd, rtm, sizeof(buf))) > 0) { + if (len < (int)sizeof(*rtm)) { + close(fd); + return (-1); + } + if (rtm->rtm_type == RTM_GET && rtm->rtm_pid == getpid() && + rtm->rtm_seq == seq) { + if (rtm->rtm_errno) { + close(fd); + errno = rtm->rtm_errno; + return (-1); + } + break; + } + } + + struct sockaddr *sa = (struct sockaddr *)(rtm + 1); + for (int i = 0; i < RTAX_MAX; i++) { + if (rtm->rtm_addrs & (1 << i)) { + if ((1 << i) == RTA_IFP) { + struct sockaddr_dl *sdl = + (struct sockaddr_dl *)sa; + if (!sdl) { + log_fatal( + "get-gateway", + "Unable to parse kernel response to request " + "for gateway MAC address. You will need to manually specify " + "your gateway MAC with the \"-G\" or \"--gateway-mac\" flag."); + } + char *_iface = xmalloc(sdl->sdl_nlen + 1); + memcpy(_iface, sdl->sdl_data, sdl->sdl_nlen); + _iface[sdl->sdl_nlen + 1] = 0; + *iface = _iface; + } + if ((1 << i) == RTA_GATEWAY) { + struct sockaddr_in *sin = + (struct sockaddr_in *)sa; + gw->s_addr = sin->sin_addr.s_addr; + } + // next element + sa = (struct sockaddr *)(ROUNDUP(sa->sa_len) + + (char *)sa); + } + } + close(fd); + return EXIT_SUCCESS; +} + +char *get_default_iface(void) +{ + struct in_addr t; + char *retv = NULL; + _get_default_gw(&t, &retv); + return retv; +} + +int get_default_gw(struct in_addr *gw, UNUSED char *iface) +{ + char *_iface = NULL; + _get_default_gw(gw, &_iface); + return EXIT_SUCCESS; +} + +#endif /* ZMAP_GET_GATEWAY_BSD_H */ diff --git a/src/get_gateway-linux.h b/src/get_gateway-linux.h new file mode 100644 index 0000000..4aac7b8 --- /dev/null +++ b/src/get_gateway-linux.h @@ -0,0 +1,322 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_GET_GATEWAY_LINUX_H +#define ZMAP_GET_GATEWAY_LINUX_H + +#ifdef ZMAP_GET_GATEWAY_BSD_H +#error "Don't include both get_gateway-bsd.h and get_gateway-linux.h" +#endif + +#include +#include +#include +#include +#include + +#define GW_BUFFER_SIZE 64000 + +int read_nl_sock(int sock, char *buf, int buf_len) +{ + int msg_len = 0; + char *pbuf = buf; + do { + int len = recv(sock, pbuf, buf_len - msg_len, 0); + if (len <= 0) { + log_debug("get-gw", "recv failed: %s", strerror(errno)); + return -1; + } + struct nlmsghdr *nlhdr = (struct nlmsghdr *)pbuf; + if (NLMSG_OK(nlhdr, ((unsigned int)len)) == 0 || + nlhdr->nlmsg_type == NLMSG_ERROR) { + log_debug("get-gw", "recv failed: %s", strerror(errno)); + return -1; + } + if (nlhdr->nlmsg_type == NLMSG_DONE) { + break; + } else { + msg_len += len; + pbuf += len; + } + if ((nlhdr->nlmsg_flags & NLM_F_MULTI) == 0) { + break; + } + } while (1); + return msg_len; +} + +int send_nl_req(uint16_t msg_type, uint32_t seq, void *payload, + uint32_t payload_len) +{ + int sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE); + if (sock < 0) { + log_error("get-gw", "unable to get socket: %s", + strerror(errno)); + return -1; + } + if (NLMSG_SPACE(payload_len) < payload_len) { + close(sock); + // Integer overflow + return -1; + } + struct nlmsghdr *nlmsg; + nlmsg = xmalloc(NLMSG_SPACE(payload_len)); + + memset(nlmsg, 0, NLMSG_SPACE(payload_len)); + memcpy(NLMSG_DATA(nlmsg), payload, payload_len); + nlmsg->nlmsg_type = msg_type; + nlmsg->nlmsg_len = NLMSG_LENGTH(payload_len); + nlmsg->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST; + nlmsg->nlmsg_seq = seq; + nlmsg->nlmsg_pid = getpid(); + + if (send(sock, nlmsg, nlmsg->nlmsg_len, 0) < 0) { + log_error("get-gw", "failure sending: %s", strerror(errno)); + return -1; + } + free(nlmsg); + return sock; +} + +int get_hw_addr(struct in_addr *gw_ip, char *iface, unsigned char *hw_mac) +{ + struct ndmsg req; + memset(&req, 0, sizeof(struct ndmsg)); + + if (!gw_ip || !hw_mac) { + return -1; + } + // Send RTM_GETNEIGH request + req.ndm_family = AF_INET; + req.ndm_ifindex = if_nametoindex(iface); + req.ndm_state = NUD_REACHABLE; + req.ndm_type = NDA_LLADDR; + + int sock = send_nl_req(RTM_GETNEIGH, 1, &req, sizeof(req)); + // Read responses + char *buf = xmalloc(GW_BUFFER_SIZE); + int nl_len = read_nl_sock(sock, buf, GW_BUFFER_SIZE); + if (nl_len <= 0) { + free(buf); + return -1; + } + // Parse responses + struct nlmsghdr *nlhdr = (struct nlmsghdr *)buf; + while (NLMSG_OK(nlhdr, nl_len)) { + struct rtattr *rt_attr; + struct rtmsg *rt_msg; + int rt_len; + unsigned char mac[6]; + struct in_addr dst_ip; + int correct_ip = 0; + + rt_msg = (struct rtmsg *)NLMSG_DATA(nlhdr); + if ((rt_msg->rtm_family != AF_INET)) { + free(buf); + return -1; + } + rt_attr = (struct rtattr *)RTM_RTA(rt_msg); + rt_len = RTM_PAYLOAD(nlhdr); + while (RTA_OK(rt_attr, rt_len)) { + switch (rt_attr->rta_type) { + case NDA_LLADDR: + if (RTA_PAYLOAD(rt_attr) != IFHWADDRLEN) { + // could be using a VPN + log_fatal( + "get_gateway", + "Unexpected hardware address length (%d)." + " If you are using a VPN, supply the --iplayer flag (and provide an" + " interface via -i)", + RTA_PAYLOAD(rt_attr)); + } + memcpy(mac, RTA_DATA(rt_attr), IFHWADDRLEN); + break; + case NDA_DST: + if (RTA_PAYLOAD(rt_attr) != sizeof(dst_ip)) { + // could be using a VPN + log_fatal( + "get_gateway", + "Unexpected IP address length (%d)." + " If you are using a VPN, supply the --iplayer flag" + " (and provide an interface via -i)", + RTA_PAYLOAD(rt_attr)); + } + memcpy(&dst_ip, RTA_DATA(rt_attr), + sizeof(dst_ip)); + if (memcmp(&dst_ip, gw_ip, sizeof(dst_ip)) == + 0) { + correct_ip = 1; + } + break; + } + rt_attr = RTA_NEXT(rt_attr, rt_len); + } + if (correct_ip) { + memcpy(hw_mac, mac, IFHWADDRLEN); + free(buf); + return 0; + } + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + } + free(buf); + return -1; +} + +// gw and iface[IF_NAMESIZE] MUST be allocated +int _get_default_gw(struct in_addr *gw, char *iface) +{ + struct rtmsg req; + unsigned int nl_len; + char buf[8192]; + struct nlmsghdr *nlhdr; + + if (!gw || !iface) { + return -1; + } + + // Send RTM_GETROUTE request + memset(&req, 0, sizeof(req)); + int sock = send_nl_req(RTM_GETROUTE, 0, &req, sizeof(req)); + + // Read responses + nl_len = read_nl_sock(sock, buf, sizeof(buf)); + if (nl_len <= 0) { + return -1; + } + + // Parse responses + nlhdr = (struct nlmsghdr *)buf; + while (NLMSG_OK(nlhdr, nl_len)) { + struct rtattr *rt_attr; + struct rtmsg *rt_msg; + int rt_len; + int has_gw = 0; + + rt_msg = (struct rtmsg *)NLMSG_DATA(nlhdr); + + // There could be multiple routing tables. Loop until we find the + // correct one. + if ((rt_msg->rtm_family != AF_INET) || + (rt_msg->rtm_table != RT_TABLE_MAIN)) { + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + continue; + } + + rt_attr = (struct rtattr *)RTM_RTA(rt_msg); + rt_len = RTM_PAYLOAD(nlhdr); + while (RTA_OK(rt_attr, rt_len)) { + switch (rt_attr->rta_type) { + case RTA_OIF: + if_indextoname(*(int *)RTA_DATA(rt_attr), + iface); + break; + case RTA_GATEWAY: + gw->s_addr = *(unsigned int *)RTA_DATA(rt_attr); + has_gw = 1; + break; + } + rt_attr = RTA_NEXT(rt_attr, rt_len); + } + + if (has_gw) { + return 0; + } + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + } + return -1; +} + +char *get_default_iface(void) +{ + struct in_addr gw; + char *iface; + + iface = malloc(IF_NAMESIZE); + memset(iface, 0, IF_NAMESIZE); + + if(_get_default_gw(&gw, iface)) { + log_fatal( + "send", + "ZMap could not detect your default network interface. " + "You likely do not have sufficient privileges to open a raw packet socket. " + "Are you running as root or with the CAP_NET_RAW capability? If you are, you " + "may need to manually set interface using the \"-i\" flag."); + } else { + return iface; + } +} + +int get_default_gw(struct in_addr *gw, char *iface) +{ + char _iface[IF_NAMESIZE]; + memset(_iface, 0, IF_NAMESIZE); + + _get_default_gw(gw, _iface); + if (strcmp(iface, _iface)) { + log_fatal( + "get-gateway", + "The specified network (\"%s\") does not match " + "the interface associated with the default gateway (%s). You will " + "need to manually specify the MAC address of your gateway using " + "the \"--gateway-mac\" flag.", + iface, _iface); + } + return EXIT_SUCCESS; +} + +int get_iface_ip(char *iface, struct in_addr *ip) +{ + int sock; + struct ifreq ifr; + memset(&ifr, 0, sizeof(struct ifreq)); + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock < 0) { + log_fatal("get-iface-ip", "failure opening socket: %s", + strerror(errno)); + } + ifr.ifr_addr.sa_family = AF_INET; + strncpy(ifr.ifr_name, iface, IFNAMSIZ - 1); + + if (ioctl(sock, SIOCGIFADDR, &ifr) < 0) { + close(sock); + log_fatal( + "get-iface-ip", + "Unable to automatically identify the correct " + "source address for %s interface. ioctl failure: %s. " + "If this is the unexpected interface, you can manually specify " + "the correct interface with \"-i\" flag. If this is the correct " + "interface, you likely need to manually specify the source IP " + "address to use with the \"-S\" flag.", + iface, strerror(errno)); + } + ip->s_addr = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr.s_addr; + close(sock); + return EXIT_SUCCESS; +} + +int get_iface_hw_addr(char *iface, unsigned char *hw_mac) +{ + int s; + struct ifreq buffer; + + // Load the hwaddr from a dummy socket + s = socket(PF_INET, SOCK_DGRAM, 0); + if (s < 0) { + log_error("get_iface_hw_addr", "Unable to open socket: %s", + strerror(errno)); + return EXIT_FAILURE; + } + memset(&buffer, 0, sizeof(buffer)); + strncpy(buffer.ifr_name, iface, IFNAMSIZ); + ioctl(s, SIOCGIFHWADDR, &buffer); + close(s); + memcpy(hw_mac, buffer.ifr_hwaddr.sa_data, 6); + return EXIT_SUCCESS; +} + +#endif /* ZMAP_GET_GATEWAY_LINUX_H */ diff --git a/src/get_gateway.c b/src/get_gateway.c new file mode 100644 index 0000000..87abfcc --- /dev/null +++ b/src/get_gateway.c @@ -0,0 +1,30 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +#include + +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__DragonFly__) +#include "get_gateway-bsd.h" +#else // (linux) +#include "get_gateway-linux.h" +#endif diff --git a/src/get_gateway.h b/src/get_gateway.h new file mode 100644 index 0000000..2df448c --- /dev/null +++ b/src/get_gateway.h @@ -0,0 +1,20 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef GET_GATEWAY_H +#define GET_GATEWAY_H + +#include + +int get_hw_addr(struct in_addr *gw_ip, char *iface, unsigned char *hw_mac); +int get_default_gw(struct in_addr *gw, char *iface); +int get_iface_ip(char *iface, struct in_addr *ip); +int get_iface_hw_addr(char *iface, unsigned char *hw_mac); +char *get_default_iface(void); + +#endif diff --git a/src/iterator.c b/src/iterator.c new file mode 100644 index 0000000..ba83db4 --- /dev/null +++ b/src/iterator.c @@ -0,0 +1,144 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/xalloc.h" +#include "../lib/util.h" + +#include "iterator.h" + +#include "aesrand.h" +#include "shard.h" +#include "state.h" + +struct iterator { + cycle_t cycle; + uint8_t num_threads; + shard_t *thread_shards; + uint8_t *complete; + pthread_mutex_t mutex; + uint32_t curr_threads; +}; + +void shard_complete(uint8_t thread_id, void *arg) +{ + iterator_t *it = (iterator_t *)arg; + assert(thread_id < it->num_threads); + pthread_mutex_lock(&it->mutex); + it->complete[thread_id] = 1; + it->curr_threads--; + shard_t *s = &it->thread_shards[thread_id]; + zsend.packets_sent += s->state.packets_sent; + zsend.targets_scanned += s->state.targets_scanned; + zsend.sendto_failures += s->state.packets_failed; + uint8_t done = 1; + for (uint8_t i = 0; done && (i < it->num_threads); ++i) { + done = done && it->complete[i]; + } + if (done) { + zsend.finish = now(); + zsend.complete = 1; + zsend.first_scanned = it->thread_shards[0].state.first_scanned; + } + pthread_mutex_unlock(&it->mutex); +} + +static uint64_t bits_needed(uint64_t n) +{ + n -= 1; + int r = 0; + while (n) { + r++; + n >>= 1; + } + return r; +} + +iterator_t *iterator_init(uint8_t num_threads, uint16_t shard, + uint16_t num_shards, uint64_t num_addrs, + uint32_t num_ports) +{ + uint8_t bits_for_ip = bits_needed(num_addrs); + log_debug("iterator", "bits needed for %u addresses: %u", num_addrs, + bits_for_ip); + uint8_t bits_for_port = bits_needed(num_ports); + log_debug("iterator", "bits needed for %u ports: %u", num_ports, + bits_for_port); + uint64_t group_min_size = ((uint64_t)1) + << (bits_for_ip + bits_for_port); + log_debug("iterator", "minimum elements to iterate over: %llu", + group_min_size); + iterator_t *it = xmalloc(sizeof(struct iterator)); + const cyclic_group_t *group = get_group(group_min_size); + if (num_addrs > (1LL << 32)) { + zsend.max_index = 0xFFFFFFFF; + } else { + zsend.max_index = (uint32_t)num_addrs; + } + log_debug("iterator", "max index %u", zsend.max_index); + it->cycle = make_cycle(group, zconf.aes); + it->num_threads = num_threads; + it->curr_threads = num_threads; + it->thread_shards = xcalloc(num_threads, sizeof(shard_t)); + it->complete = xcalloc(it->num_threads, sizeof(uint8_t)); + pthread_mutex_init(&it->mutex, NULL); + log_debug("iterator", "max targets is %u", zsend.max_targets); + for (uint8_t i = 0; i < num_threads; ++i) { + shard_init(&it->thread_shards[i], shard, num_shards, i, + num_threads, zsend.max_targets, bits_for_port, + &it->cycle, shard_complete, it); + } + zconf.generator = it->cycle.generator; + return it; +} + +uint64_t iterator_get_sent(iterator_t *it) +{ + uint64_t sent = 0; + for (uint8_t i = 0; i < it->num_threads; ++i) { + sent += it->thread_shards[i].state.packets_sent; + } + return sent; +} + +uint64_t iterator_get_iterations(iterator_t *it) +{ + uint64_t iterations = 0; + for (uint8_t i = 0; i < it->num_threads; ++i) { + iterations += it->thread_shards[i].iterations; + } + return iterations; +} + +uint32_t iterator_get_fail(iterator_t *it) +{ + uint32_t fails = 0; + for (uint8_t i = 0; i < it->num_threads; ++i) { + fails += it->thread_shards[i].state.packets_failed; + } + return fails; +} + +shard_t *get_shard(iterator_t *it, uint8_t thread_id) +{ + assert(thread_id < it->num_threads); + return &it->thread_shards[thread_id]; +} + +uint32_t iterator_get_curr_send_threads(iterator_t *it) +{ + assert(it); + return it->curr_threads; +} diff --git a/src/iterator.h b/src/iterator.h new file mode 100644 index 0000000..c455992 --- /dev/null +++ b/src/iterator.h @@ -0,0 +1,34 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_ITERATOR_H +#define ZMAP_ITERATOR_H + +#include + +#include "../lib/includes.h" + +#include "aesrand.h" +#include "cyclic.h" +#include "shard.h" + +typedef struct iterator iterator_t; + +iterator_t *iterator_init(uint8_t num_threads, uint16_t shard, + uint16_t num_shards, uint64_t num_addrs, + uint32_t num_ports); + +uint64_t iterator_get_sent(iterator_t *it); +uint64_t iterator_get_iterations(iterator_t *it); +uint32_t iterator_get_fail(iterator_t *it); + +uint32_t iterator_get_curr_send_threads(iterator_t *it); + +shard_t *get_shard(iterator_t *it, uint8_t thread_id); + +#endif /* ZMAP_ITERATOR_H */ diff --git a/src/lexer.l b/src/lexer.l new file mode 100644 index 0000000..48e0ad2 --- /dev/null +++ b/src/lexer.l @@ -0,0 +1,36 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +%{ +#pragma GCC diagnostic ignored "-Wredundant-decls" +#pragma GCC diagnostic ignored "-Wmissing-noreturn" + +#include +#include "parser.h" + +%} + +%option noinput +%option nounput +%% +[0-9]+ yylval.int_literal = (uint64_t) atoll(yytext); return T_NUMBER; +\n /* Ignore end of line */ +[ \t]+ /* Ignore whitespace */ +!= return T_NOT_EQ; +>= return T_GT_EQ; +"<=" return T_LT_EQ; +&& return T_AND; +"||" return T_OR; += return '='; +">" return '>'; +"<" return '<'; +"(" return '('; +")" return ')'; +[a-zA-Z][-_a-zA-Z0-9]+ yylval.string_literal = strdup(yytext); return T_FIELD; + +%% diff --git a/src/monitor.c b/src/monitor.c new file mode 100644 index 0000000..ee9ac9b --- /dev/null +++ b/src/monitor.c @@ -0,0 +1,491 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// module responsible for printing on-screen updates during the scan process + +#include "monitor.h" + +#define __STDC_FORMAT_MACROS +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../lib/lockfd.h" +#include "../lib/logger.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#include "iterator.h" +#include "recv.h" +#include "state.h" + +#define UPDATE_INTERVAL 1 // seconds +#define NUMBER_STR_LEN 20 +#define WARMUP_PERIOD 5 +#define MIN_HITRATE_TIME_WINDOW 5 // seconds + +// internal monitor status that is used to track deltas +typedef struct internal_scan_status { + double last_now; + uint64_t last_sent; + uint64_t last_tried_sent; + uint32_t last_send_failures; + uint32_t last_recv_net_success; + uint32_t last_recv_app_success; + uint32_t last_recv_total; + uint32_t last_pcap_drop; + double min_hitrate_start; +} int_status_t; + +// exportable status information that can be printed to screen +typedef struct export_scan_status { + uint64_t total_sent; + uint64_t total_tried_sent; + uint32_t recv_success_unique; + uint32_t app_recv_success_unique; + uint64_t total_recv; + uint32_t complete; + uint32_t send_threads; + double percent_complete; + + double hitrate; // network, e.g. SYN-ACK vs RST + double + app_hitrate; // application level, e.g. DNS response versus correct + // lookup. + + double send_rate; + char send_rate_str[NUMBER_STR_LEN]; + double send_rate_avg; + char send_rate_avg_str[NUMBER_STR_LEN]; + + double recv_rate; + char recv_rate_str[NUMBER_STR_LEN]; + double recv_avg; + char recv_avg_str[NUMBER_STR_LEN]; + double recv_total_rate; + double recv_total_avg; + + double app_success_rate; + char app_success_rate_str[NUMBER_STR_LEN]; + double app_success_avg; + char app_success_avg_str[NUMBER_STR_LEN]; + + uint32_t pcap_drop; + uint32_t pcap_ifdrop; + uint32_t pcap_drop_total; + char pcap_drop_total_str[NUMBER_STR_LEN]; + double pcap_drop_last; + char pcap_drop_last_str[NUMBER_STR_LEN]; + double pcap_drop_avg; + char pcap_drop_avg_str[NUMBER_STR_LEN]; + + uint32_t time_remaining; + char time_remaining_str[NUMBER_STR_LEN]; + uint32_t time_past; + char time_past_str[NUMBER_STR_LEN]; + + uint32_t fail_total; + double fail_avg; + double fail_last; + float seconds_under_min_hitrate; + +} export_status_t; + +static FILE *status_fd = NULL; + +// find minimum of an array of doubles +static double min_d(double array[], int n) +{ + double value = INFINITY; + for (int i = 0; i < n; i++) { + if (array[i] < value) { + value = array[i]; + } + } + return value; +} + +// estimate time remaining time based on config and state +double compute_remaining_time(double age, uint64_t packets_sent, + uint64_t iterations) +{ + if (!zsend.complete) { + double remaining[] = {INFINITY, INFINITY, INFINITY, INFINITY, + INFINITY}; + if (zsend.list_of_ips_pbm) { + // Estimate progress using group iterations + double done = + (double)iterations / + ((uint64_t)0xFFFFFFFFU / zconf.total_shards); + remaining[0] = + (1. - done) * (age / done) + zconf.cooldown_secs; + } + if (zsend.max_targets) { + double done = + (double)packets_sent / + ((uint64_t)zsend.max_targets * + zconf.packet_streams / zconf.total_shards); + remaining[1] = + (1. - done) * (age / done) + zconf.cooldown_secs; + } + if (zconf.max_runtime) { + remaining[2] = + (zconf.max_runtime - age) + zconf.cooldown_secs; + } + if (zconf.max_results) { + double done = + (double)zrecv.filter_success / zconf.max_results; + remaining[3] = (1. - done) * (age / done); + } + if (zsend.max_index) { + double done = + (double)packets_sent / + ((uint64_t)zsend.max_index * zconf.packet_streams / + zconf.total_shards); + remaining[4] = + (1. - done) * (age / done) + zconf.cooldown_secs; + } + return min_d(remaining, sizeof(remaining) / sizeof(double)); + } else { + return zconf.cooldown_secs - (now() - zsend.finish); + } +} + +static void update_pcap_stats(pthread_mutex_t *recv_ready_mutex) +{ + // ask pcap for fresh values + pthread_mutex_lock(recv_ready_mutex); + recv_update_stats(); + pthread_mutex_unlock(recv_ready_mutex); +} + +static void export_stats(int_status_t *intrnl, export_status_t *exp, + iterator_t *it) +{ + uint64_t total_sent = iterator_get_sent(it); + uint64_t total_iterations = iterator_get_iterations(it); + uint32_t total_fail = iterator_get_fail(it); + uint64_t total_recv = zrecv.pcap_recv; + uint64_t recv_success = zrecv.filter_success; + uint32_t app_success = zrecv.app_success_unique; + double cur_time = now(); + double age = cur_time - zsend.start; // time of entire scan + // time since the last time we updated + double delta = cur_time - intrnl->last_now; + double remaining_secs = + compute_remaining_time(age, total_sent, total_iterations); + + // export amount of time the scan has been running + if (age < WARMUP_PERIOD) { + exp->time_remaining_str[0] = '\0'; + } else { + char buf[20]; + time_string(ceil(remaining_secs), 1, buf, sizeof(buf)); + snprintf(exp->time_remaining_str, NUMBER_STR_LEN, " (%s left)", + buf); + } + exp->time_past = age; + exp->time_remaining = remaining_secs; + time_string((int)age, 0, exp->time_past_str, NUMBER_STR_LEN); + + // export recv statistics + exp->recv_rate = + ceil((recv_success - intrnl->last_recv_net_success) / delta); + number_string(exp->recv_rate, exp->recv_rate_str, NUMBER_STR_LEN); + exp->recv_avg = recv_success / age; + number_string(exp->recv_avg, exp->recv_avg_str, NUMBER_STR_LEN); + exp->recv_total_rate = (total_recv - intrnl->last_recv_total) / delta; + exp->recv_total_avg = total_recv / age; + + // application level statistics + if (zconf.fsconf.app_success_index >= 0) { + exp->app_success_rate = + (app_success - intrnl->last_recv_app_success) / delta; + number_string(exp->app_success_rate, exp->app_success_rate_str, + NUMBER_STR_LEN); + exp->app_success_avg = (app_success / age); + number_string(exp->app_success_avg, exp->app_success_avg_str, + NUMBER_STR_LEN); + } + + if (!total_sent) { + exp->hitrate = 0; + exp->app_hitrate = 0; + } else { + exp->hitrate = recv_success * 100.0 / total_sent; + exp->app_hitrate = app_success * 100.0 / total_sent; + } + + if (age > WARMUP_PERIOD && exp->hitrate < zconf.min_hitrate) { + if (fabs(intrnl->min_hitrate_start) < .00001) { + intrnl->min_hitrate_start = cur_time; + } + } else { + intrnl->min_hitrate_start = 0.0; + } + if (fabs(intrnl->min_hitrate_start) < .00001) { + exp->seconds_under_min_hitrate = 0; + } else { + exp->seconds_under_min_hitrate = + cur_time - intrnl->min_hitrate_start; + } + if (!zsend.complete) { + exp->send_rate = ceil((total_sent - intrnl->last_sent) / delta); + number_string(exp->send_rate, exp->send_rate_str, + NUMBER_STR_LEN); + exp->send_rate_avg = total_sent / age; + number_string(exp->send_rate_avg, exp->send_rate_avg_str, + NUMBER_STR_LEN); + } else { + exp->send_rate_avg = total_sent / (zsend.finish - zsend.start); + number_string(exp->send_rate_avg, exp->send_rate_avg_str, + NUMBER_STR_LEN); + } + // export other pre-calculated values + exp->total_sent = total_sent; + exp->total_tried_sent = total_iterations; + exp->percent_complete = 100. * age / (age + remaining_secs); + exp->recv_success_unique = recv_success; + exp->app_recv_success_unique = app_success; + exp->total_recv = total_recv; + exp->complete = zsend.complete; + + // pcap dropped packets + exp->pcap_drop = zrecv.pcap_drop; + exp->pcap_ifdrop = zrecv.pcap_ifdrop; + exp->pcap_drop_total = exp->pcap_drop + exp->pcap_ifdrop; + exp->pcap_drop_last = + (exp->pcap_drop_total - intrnl->last_pcap_drop) / delta; + exp->pcap_drop_avg = exp->pcap_drop_total / age; + number_string(exp->pcap_drop_total, exp->pcap_drop_total_str, + NUMBER_STR_LEN); + number_string(exp->pcap_drop_last, exp->pcap_drop_last_str, + NUMBER_STR_LEN); + number_string(exp->pcap_drop_avg, exp->pcap_drop_avg_str, + NUMBER_STR_LEN); + + zsend.sendto_failures = total_fail; + exp->fail_total = zsend.sendto_failures; + exp->fail_last = (exp->fail_total - intrnl->last_send_failures) / delta; + exp->fail_avg = exp->fail_total / age; + + // misc + exp->send_threads = iterator_get_curr_send_threads(it); + + // Update internal stats + intrnl->last_now = cur_time; + intrnl->last_sent = exp->total_sent; + intrnl->last_recv_net_success = exp->recv_success_unique; + intrnl->last_recv_app_success = exp->app_recv_success_unique; + intrnl->last_pcap_drop = exp->pcap_drop_total; + intrnl->last_send_failures = exp->fail_total; + intrnl->last_recv_total = exp->total_recv; +} + +static void log_drop_warnings(export_status_t *exp) +{ + if (exp->pcap_drop_last / exp->recv_rate > 0.05) { + log_warn("monitor", + "Dropped %.0f packets in the last second, (%u total " + "dropped (pcap: %u + iface: %u))", + exp->pcap_drop_last, exp->pcap_drop_total, + exp->pcap_drop, exp->pcap_ifdrop); + } + if (exp->fail_last / exp->send_rate > 0.01) { + log_warn("monitor", + "Failed to send %.0f packets/sec (%u total failures)", + exp->fail_last, exp->fail_total); + } +} + +static void onscreen_appsuccess(export_status_t *exp) +{ + // this when probe module handles application-level success rates + if (!exp->complete) { + fprintf(stderr, + "%5s %0.0f%%%s; sent: %" PRIu64 " %sp/s (%sp/s avg); " + "recv: %u %sp/s (%sp/s avg); " + "app success: %u %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%% " + "app hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_str, exp->send_rate_avg_str, + exp->recv_success_unique, exp->recv_rate_str, + exp->recv_avg_str, exp->app_recv_success_unique, + exp->app_success_rate_str, exp->app_success_avg_str, + exp->pcap_drop_last_str, exp->pcap_drop_avg_str, + exp->hitrate, exp->app_hitrate); + } else { + fprintf(stderr, + "%5s %0.0f%%%s; sent: %" PRIu64 " done (%sp/s avg); " + "recv: %u %sp/s (%sp/s avg); " + "app success: %u %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%% " + "app hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_avg_str, exp->recv_success_unique, + exp->recv_rate_str, exp->recv_avg_str, + exp->app_recv_success_unique, exp->app_success_rate_str, + exp->app_success_avg_str, exp->pcap_drop_last_str, + exp->pcap_drop_avg_str, exp->hitrate, exp->app_hitrate); + } +} + +static void onscreen_generic(export_status_t *exp) +{ + if (!exp->complete) { + fprintf(stderr, + "%5s %0.0f%%%s; send: %" PRIu64 " %sp/s (%sp/s avg); " + "recv: %u %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_str, exp->send_rate_avg_str, + exp->recv_success_unique, exp->recv_rate_str, + exp->recv_avg_str, exp->pcap_drop_last_str, + exp->pcap_drop_avg_str, exp->hitrate); + } else { + fprintf(stderr, + "%5s %0.0f%%%s; send: %" PRIu64 " done (%sp/s avg); " + "recv: %u %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_avg_str, exp->recv_success_unique, + exp->recv_rate_str, exp->recv_avg_str, + exp->pcap_drop_last_str, exp->pcap_drop_avg_str, + exp->hitrate); + } + fflush(stderr); +} + +static FILE *init_status_update_file(char *path) +{ + FILE *f = fopen(path, "w"); + if (!f) { + log_fatal("csv", "could not open status updates file (%s): %s", + zconf.status_updates_file, strerror(errno)); + } + log_debug("monitor", "status updates CSV will be saved to %s", + zconf.status_updates_file); + fprintf( + f, + "real-time,time-elapsed,time-remaining," + "percent-complete,hit-rate,active-send-threads," + "sent-total,sent-last-one-sec,sent-avg-per-sec," + "recv-success-total,recv-success-last-one-sec,recv-success-avg-per-sec," + "recv-total,recv-total-last-one-sec,recv-total-avg-per-sec," + "pcap-drop-total,drop-last-one-sec,drop-avg-per-sec," + "sendto-fail-total,sendto-fail-last-one-sec,sendto-fail-avg-per-sec\n"); + fflush(f); + return f; +} + +static void update_status_updates_file(export_status_t *exp, FILE *f) +{ + struct timeval now; + char timestamp[256]; + gettimeofday(&now, NULL); + time_t sec = now.tv_sec; + struct tm *ptm = localtime(&sec); + strftime(timestamp, 20, "%Y-%m-%d %H:%M:%S", ptm); + + fprintf(f, + "%s,%u,%u," + "%f,%f,%u," + "%" PRIu64 ",%.0f,%.0f," + "%u,%.0f,%.0f," + "%" PRIu64 ",%.0f,%.0f," + "%u,%.0f,%.0f," + "%u,%.0f,%.0f\n", + timestamp, exp->time_past, exp->time_remaining, + exp->percent_complete, exp->hitrate, exp->send_threads, + exp->total_sent, exp->send_rate, exp->send_rate_avg, + exp->recv_success_unique, exp->recv_rate, exp->recv_avg, + exp->total_recv, exp->recv_total_rate, exp->recv_total_avg, + exp->pcap_drop_total, exp->pcap_drop_last, exp->pcap_drop_avg, + exp->fail_total, exp->fail_last, exp->fail_avg); + fflush(f); +} + +static inline void check_min_hitrate(export_status_t *exp) +{ + if (exp->seconds_under_min_hitrate >= MIN_HITRATE_TIME_WINDOW) { + log_fatal("monitor", + "hitrate below %.0f for %.0f seconds. aborting scan.", + zconf.min_hitrate, exp->seconds_under_min_hitrate); + } +} + +static inline void check_max_sendto_failures(export_status_t *exp) +{ + if (zconf.max_sendto_failures >= 0 && + exp->fail_total > (uint32_t)zconf.max_sendto_failures) { + log_fatal("monitor", + "maximum number of sendto failures (%i) exceeded", + zconf.max_sendto_failures); + } +} + +void monitor_init(void) +{ + if (zconf.status_updates_file) { + status_fd = init_status_update_file(zconf.status_updates_file); + assert(status_fd); + } +} + +void monitor_run(iterator_t *it, pthread_mutex_t *lock) +{ + int_status_t *internal_status = xmalloc(sizeof(int_status_t)); + export_status_t *export_status = xmalloc(sizeof(export_status_t)); + + while (!(zsend.complete && zrecv.complete)) { + update_pcap_stats(lock); + export_stats(internal_status, export_status, it); + log_drop_warnings(export_status); + check_min_hitrate(export_status); + check_max_sendto_failures(export_status); + if (!zconf.quiet) { + lock_file(stderr); + if (zconf.fsconf.app_success_index >= 0) { + onscreen_appsuccess(export_status); + } else { + onscreen_generic(export_status); + } + unlock_file(stderr); + } + if (status_fd) { + update_status_updates_file(export_status, status_fd); + } + sleep(UPDATE_INTERVAL); + } + if (!zconf.quiet) { + lock_file(stderr); + fflush(stderr); + unlock_file(stderr); + } + if (status_fd) { + fflush(status_fd); + fclose(status_fd); + } +} diff --git a/src/monitor.h b/src/monitor.h new file mode 100644 index 0000000..6aa564c --- /dev/null +++ b/src/monitor.h @@ -0,0 +1,18 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include "iterator.h" + +#ifndef MONITOR_H +#define MONITOR_H + +void monitor_run(iterator_t *it, pthread_mutex_t *lock); +void monitor_init(void); + +#endif diff --git a/src/output_modules/module_csv.c b/src/output_modules/module_csv.c new file mode 100644 index 0000000..d241964 --- /dev/null +++ b/src/output_modules/module_csv.c @@ -0,0 +1,126 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../../lib/logger.h" +#include "../fieldset.h" + +#include "output_modules.h" + +static FILE *file = NULL; + +int csv_init(struct state_conf *conf, const char **fields, int fieldlens) +{ + assert(conf); + if (conf->output_filename) { + if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + log_fatal( + "csv", + "could not open CSV output file (%s): %s", + conf->output_filename, strerror(errno)); + } + } + } else { + file = stdout; + log_debug("csv", "no output file selected, will use stdout"); + } + if (!conf->no_header_row) { + log_debug("csv", "more than one field, will add headers"); + for (int i = 0; i < fieldlens; i++) { + if (i) { + fprintf(file, ","); + } + fprintf(file, "%s", fields[i]); + } + fprintf(file, "\n"); + } + check_and_log_file_error(file, "csv"); + return EXIT_SUCCESS; +} + +int csv_close(__attribute__((unused)) struct state_conf *c, + __attribute__((unused)) struct state_send *s, + __attribute__((unused)) struct state_recv *r) +{ + if (file) { + fflush(file); + fclose(file); + } + return EXIT_SUCCESS; +} + +static void hex_encode(FILE *f, unsigned char *readbuf, size_t len) +{ + for (size_t i = 0; i < len; i++) { + fprintf(f, "%02x", readbuf[i]); + } + check_and_log_file_error(f, "csv"); +} + +int csv_process(fieldset_t *fs) +{ + if (!file) { + return EXIT_SUCCESS; + } + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + if (i) { + fprintf(file, ","); + } + if (f->type == FS_STRING) { + if (strchr((char *)f->value.ptr, ',')) { + fprintf(file, "\"%s\"", (char *)f->value.ptr); + } else { + fprintf(file, "%s", (char *)f->value.ptr); + } + } else if (f->type == FS_UINT64) { + fprintf(file, "%" PRIu64, (uint64_t)f->value.num); + } else if (f->type == FS_BOOL) { + fprintf(file, "%" PRIi32, (int)f->value.num); + } else if (f->type == FS_BINARY) { + hex_encode(file, (unsigned char *)f->value.ptr, f->len); + } else if (f->type == FS_NULL) { + // do nothing + } else { + log_fatal("csv", "received unknown output type"); + } + } + fprintf(file, "\n"); + fflush(file); + check_and_log_file_error(file, "csv"); + return EXIT_SUCCESS; +} + +output_module_t module_csv_file = { + .name = "csv", + .init = &csv_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &csv_close, + .process_ip = &csv_process, + .supports_dynamic_output = NO_DYNAMIC_SUPPORT, + .helptext = + "Outputs one or more output fields as a comma-delimited file. By default, the " + "probe module does not filter out duplicates or limit to successful fields, " + "but rather includes all received packets. Fields can be controlled by " + "setting --output-fields. Filtering out failures and duplicate packets can " + "be achieved by setting an --output-filter."}; diff --git a/src/output_modules/module_csv.h b/src/output_modules/module_csv.h new file mode 100644 index 0000000..38a1958 --- /dev/null +++ b/src/output_modules/module_csv.h @@ -0,0 +1,14 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "../fieldset.h" +#include "output_modules.h" + +int csv_init(struct state_conf *conf, char **fields, int fieldlens); +int csv_process(fieldset_t *fs); +int csv_close(struct state_conf *c, struct state_send *s, struct state_recv *r); diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c new file mode 100644 index 0000000..a064453 --- /dev/null +++ b/src/output_modules/module_json.c @@ -0,0 +1,162 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../../lib/logger.h" + +#include "output_modules.h" +#include "../probe_modules/probe_modules.h" + +static FILE *file = NULL; + +int json_output_file_init(struct state_conf *conf, UNUSED const char **fields, + UNUSED int fieldlens) +{ + assert(conf); + if (!conf->output_filename) { + file = stdout; + } else if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + log_fatal("output-json", + "could not open JSON output file (%s): %s", + conf->output_filename, strerror(errno)); + } + } + check_and_log_file_error(file, "json"); + return EXIT_SUCCESS; +} + +char *hex_encode(unsigned char *packet, int buflen) +{ + char *buf = xmalloc(2 * buflen + 1); + for (int i = 0; i < buflen; i++) { + snprintf(buf + (i * 2), 3, "%.2x", packet[i]); + } + buf[buflen * 2] = 0; + return buf; +} + +json_object *fs_to_jsonobj(fieldset_t *fs); +json_object *repeated_to_jsonobj(fieldset_t *fs); + +json_object *field_to_jsonobj(field_t *f) +{ + if (f->type == FS_STRING) { + return json_object_new_string((char *)f->value.ptr); + } else if (f->type == FS_UINT64) { + return json_object_new_int64(f->value.num); + } else if (f->type == FS_BOOL) { + return json_object_new_boolean(f->value.num); + } else if (f->type == FS_BINARY) { + char *encoded = hex_encode(f->value.ptr, f->len); + json_object *t = json_object_new_string(encoded); + free(encoded); + return t; + } else if (f->type == FS_NULL) { + return NULL; + } else if (f->type == FS_FIELDSET) { + return fs_to_jsonobj((fieldset_t *)f->value.ptr); + } else if (f->type == FS_REPEATED) { + return repeated_to_jsonobj((fieldset_t *)f->value.ptr); + } else { + log_fatal("json", "received unknown output type: %i", f->type); + } +} + +json_object *repeated_to_jsonobj(fieldset_t *fs) +{ + json_object *obj = json_object_new_array(); + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + json_object_array_add(obj, field_to_jsonobj(f)); + } + return obj; +} + +json_object *fs_to_jsonobj(fieldset_t *fs) +{ + json_object *obj = json_object_new_object(); + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + if (f->type != FS_NULL) { + json_object_object_add(obj, f->name, + field_to_jsonobj(f)); + } + } + return obj; +} + +int json_output_to_file(fieldset_t *fs) +{ + if (!file) { + return EXIT_SUCCESS; + } + json_object *record = fs_to_jsonobj(fs); + fprintf(file, "%s\n", + json_object_to_json_string_ext(record, JSON_C_TO_STRING_PLAIN)); + fflush(file); + check_and_log_file_error(file, "json"); + json_object_put(record); + return EXIT_SUCCESS; +} + +int json_output_file_close(UNUSED struct state_conf *c, + UNUSED struct state_send *s, + UNUSED struct state_recv *r) +{ + if (file) { + fflush(file); + fclose(file); + } + return EXIT_SUCCESS; +} + +int print_json_fieldset(fieldset_t *fs) +{ + json_object *record = fs_to_jsonobj(fs); + fprintf(stdout, "%s\n", json_object_to_json_string(record)); + json_object_put(record); + return EXIT_SUCCESS; +} + +output_module_t module_json_file = { + .name = "json", + .init = &json_output_file_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &json_output_file_close, + .process_ip = &json_output_to_file, + .supports_dynamic_output = DYNAMIC_SUPPORT, + .helptext = + "Outputs one or more output fields as a json valid file. By default, the \n" + "probe module does not filter out duplicates or limit to successful fields, \n" + "but rather includes all received packets. Fields can be controlled by \n" + "setting --output-fields. Filtering out failures and duplicate packets can \n" + "be achieved by setting an --output-filter."}; diff --git a/src/output_modules/module_json.h b/src/output_modules/module_json.h new file mode 100644 index 0000000..10526a1 --- /dev/null +++ b/src/output_modules/module_json.h @@ -0,0 +1,19 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fieldset.h" + +int print_json_fieldset(fieldset_t *fs); diff --git a/src/output_modules/output_modules.c b/src/output_modules/output_modules.c new file mode 100644 index 0000000..a8d5d71 --- /dev/null +++ b/src/output_modules/output_modules.c @@ -0,0 +1,41 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include + +#include "output_modules.h" + +extern output_module_t module_csv_file; +extern output_module_t module_json_file; + +output_module_t *output_modules[] = { + &module_csv_file, &module_json_file, + // ADD YOUR MODULE HERE +}; + +output_module_t *get_output_module_by_name(const char *name) +{ + int num_modules = + (int)(sizeof(output_modules) / sizeof(output_modules[0])); + for (int i = 0; i < num_modules; i++) { + if (!strcmp(output_modules[i]->name, name)) { + return output_modules[i]; + } + } + return NULL; +} + +void print_output_modules(void) +{ + int num_modules = + (int)(sizeof(output_modules) / sizeof(output_modules[0])); + for (int i = 0; i < num_modules; i++) { + printf("%s\n", output_modules[i]->name); + } +} diff --git a/src/output_modules/output_modules.h b/src/output_modules/output_modules.h new file mode 100644 index 0000000..87f24f9 --- /dev/null +++ b/src/output_modules/output_modules.h @@ -0,0 +1,45 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef OUTPUT_MODULES_H +#define OUTPUT_MODULES_H + +#include "../state.h" +#include "../fieldset.h" + +#define NO_DYNAMIC_SUPPORT 0 +#define DYNAMIC_SUPPORT 1 + +// called at scanner initialization +typedef int (*output_init_cb)(struct state_conf *, const char **fields, + int fieldslen); + +// called on packet receipt +typedef int (*output_packet_cb)(fieldset_t *fs); + +// called periodically during the scan +typedef int (*output_update_cb)(struct state_conf *, struct state_send *, + struct state_recv *); + +typedef struct output_module { + const char *name; + int supports_dynamic_output; + unsigned update_interval; + output_init_cb init; + output_update_cb start; + output_update_cb update; + output_update_cb close; + output_packet_cb process_ip; + const char *helptext; +} output_module_t; + +output_module_t *get_output_module_by_name(const char *); + +void print_output_modules(void); + +#endif // HEADER_OUTPUT_MODULES_H diff --git a/src/parser.y b/src/parser.y new file mode 100644 index 0000000..8a8ca64 --- /dev/null +++ b/src/parser.y @@ -0,0 +1,144 @@ +%{ +#include +#include +#include "expression.h" +#include "lexer.h" +#include "filter.h" + +void yyerror(const char *str) +{ + fprintf(stderr,"Parse error: %s\n",str); +} + +int yywrap(void) +{ + return 1; +} + +extern node_t *zfilter; + +%} + +%union { + int int_literal; + char *string_literal; + struct node_st *expr; +} + +%token '(' ')' T_AND T_OR +%token T_NUMBER +%token T_FIELD +%token T_NOT_EQ T_GT_EQ '>' '<' '=' T_LT_EQ + +%left T_OR +%left T_AND + +%type filter +%type number_filter +%type string_filter +%type filter_expr + + +%% + +expression: filter_expr + { + zfilter = $1; + } + + +filter_expr: + filter_expr T_OR filter_expr + { + $$ = make_op_node(OR); + $$->left_child = $1; + $$->right_child = $3; + } + | filter_expr T_AND filter_expr + { + $$ = make_op_node(AND); + $$->left_child = $1; + $$->right_child = $3; + } + | '(' filter_expr ')' + { + $$ = $2; + } + | filter + { + $$ = $1; + } + ; + +filter: number_filter + { + $$ = $1; + } + | string_filter + { + $$ = $1; + } + ; + +number_filter: T_FIELD '=' T_NUMBER + { + $$ = make_op_node(EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD '>' T_NUMBER + { + $$ = make_op_node(GT); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD '<' T_NUMBER + { + $$ = make_op_node(LT); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_NOT_EQ T_NUMBER + { + $$ = make_op_node(NEQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_GT_EQ T_NUMBER + { + $$ = make_op_node(GT_EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_LT_EQ T_NUMBER + { + $$ = make_op_node(LT_EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + ; + +string_filter: + T_FIELD '=' T_FIELD + { + $$ = make_op_node(EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_string_node($3); + } + | + T_FIELD T_NOT_EQ T_FIELD + { + $$ = make_op_node(NEQ); + $$->left_child = make_field_node($1); + $$->right_child = make_string_node($3); + } + ; + +%% + + diff --git a/src/ports.c b/src/ports.c new file mode 100644 index 0000000..8be081a --- /dev/null +++ b/src/ports.c @@ -0,0 +1,63 @@ +/* + * ZMap Copyright 2023 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * ZIterate is a simple utility that will iteratate over the IPv4 + * space in a pseudo-random fashion, utilizing the sharding capabilities * of + * ZMap. + */ + +#define _GNU_SOURCE + +#include + +#include "state.h" +#include "../lib/pbm.h" +#include "../lib/logger.h" + +static void add_port(struct port_conf *ports, int port) +{ + if (port < 0 || port > 0xFFFF) { + log_fatal("ports", "invalid target port specified: %i", port); + } + ports->ports[ports->port_count] = port; + if (ports->port_bitmap) { + bm_set(ports->port_bitmap, port); + } + ports->port_count++; +} + +void parse_ports(char *portdef, struct port_conf *ports) +{ + if (!strcmp(portdef, "*")) { + for (int i = 0; i <= 0xFFFF; i++) { + add_port(ports, i); + } + return; + } + char *next = strtok(portdef, ","); + while (next != NULL) { + char *dash = strchr(next, '-'); + if (dash) { // range + *dash = '\0'; + int first = atoi(next); + int last = atoi(dash + 1); + if (last > 0xFFFF) { + log_fatal("ports", + "invalid target port specified: %i", + last); + } + for (int i = first; i <= last; i++) { + add_port(ports, i); + } + } else { + add_port(ports, atoi(next)); + } + next = strtok(NULL, ","); + } +} diff --git a/src/ports.h b/src/ports.h new file mode 100644 index 0000000..294a275 --- /dev/null +++ b/src/ports.h @@ -0,0 +1,3 @@ +#include "state.h" + +void parse_ports(char *portdef, struct port_conf *ports); diff --git a/src/probe_modules/module_bacnet.c b/src/probe_modules/module_bacnet.c new file mode 100644 index 0000000..6e092ec --- /dev/null +++ b/src/probe_modules/module_bacnet.c @@ -0,0 +1,195 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "packet.h" +#include "probe_modules.h" +#include "module_bacnet.h" +#include "module_udp.h" + +#define ICMP_UNREACH_HEADER_SIZE 8 + +#define ZMAP_BACNET_PACKET_LEN \ + (sizeof(struct ether_header) + sizeof(struct ip) + \ + sizeof(struct udphdr) + 0x11) + +probe_module_t module_bacnet; + +static int num_ports; + +static uint8_t bacnet_body[] = {0x0c, 0x02, 0x3f, 0xff, 0xff, 0x19, 0x4b}; +#define BACNET_BODY_LEN 7 + +static inline uint8_t get_invoke_id(uint32_t *validation) +{ + return (uint8_t)((validation[1] >> 24) & 0xFF); +} + +int bacnet_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, void **arg) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + struct bacnet_probe *bnp = (struct bacnet_probe *)&udp_header[1]; + uint8_t *body = (uint8_t *)&bnp[1]; + + make_eth_header(eth_header, src, gw); + + uint16_t ip_len = sizeof(struct ip) + sizeof(struct udphdr) + 0x11; + assert(ip_len <= MAX_PACKET_SIZE); + make_ip_header(ip_header, IPPROTO_UDP, htons(ip_len)); + + uint16_t udp_len = sizeof(struct udphdr) + 0x11; + make_udp_header(udp_header, udp_len); + + bnp->vlc.type = ZMAP_BACNET_TYPE_IP; + bnp->vlc.function = ZMAP_BACNET_FUNCTION_UNICAST_NPDU; + bnp->vlc.length = htons(0x11); + + bnp->npdu.version = ZMAP_BACNET_NPDU_VERSION_ASHRAE_135_1995; + bnp->npdu.control = 0x04; + + bnp->apdu.type_flags = 0x00; + bnp->apdu.max_segments_apdu = 0x05; + bnp->apdu.server_choice = 0x0c; + memcpy(body, bacnet_body, BACNET_BODY_LEN); + + uint32_t seed = aesrand_getword(zconf.aes); + aesrand_t *aes = aesrand_init_from_seed(seed); + *arg = aes; + + return EXIT_SUCCESS; +} + +int bacnet_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, port_n_t dport, uint8_t ttl, + uint32_t *validation, int probe_num, UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)&ip_header[1]; + struct bacnet_probe *bnp = (struct bacnet_probe *)&udp_header[1]; + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + ip_header->ip_sum = 0; + + udp_header->uh_sport = + htons(get_src_port(num_ports, probe_num, validation)); + udp_header->uh_dport = dport; + + bnp->apdu.invoke_id = get_invoke_id(validation); + + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + *buf_len = ZMAP_BACNET_PACKET_LEN; + + return EXIT_SUCCESS; +} + +int bacnet_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation, + const struct port_conf *ports) +{ + // this will reject packets that aren't UDP or ICMP and fully process ICMP + // packets + if (udp_do_validate_packet(ip_hdr, len, src_ip, validation, num_ports, + SRC_PORT_VALIDATION, + ports) == PACKET_INVALID) { + return PACKET_INVALID; + } + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + if (!udp) { + return PACKET_INVALID; + } + const size_t min_len = + sizeof(struct udphdr) + sizeof(struct bacnet_vlc); + if (udp->uh_ulen < min_len) { + return PACKET_INVALID; + } + struct bacnet_vlc *vlc = + (struct bacnet_vlc *)get_udp_payload(udp, len); + if (vlc->type != ZMAP_BACNET_TYPE_IP) { + return PACKET_INVALID; + } + } + return PACKET_VALID; +} + +void bacnet_process_packet(const u_char *packet, uint32_t len, fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = get_ip_header(packet, len); + assert(ip_hdr); + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + assert(udp); + fs_add_uint64(fs, "sport", ntohs(udp->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp->uh_dport)); + fs_add_constchar(fs, "classification", "bacnet"); + fs_add_bool(fs, "success", 1); + fs_add_null_icmp(fs); + uint32_t udp_offset = + sizeof(struct ether_header) + ip_hdr->ip_hl * 4; + uint32_t payload_offset = udp_offset + sizeof(struct udphdr); + assert(payload_offset < len); + uint8_t *payload = get_udp_payload(udp, len); + uint32_t payload_len = len - payload_offset; + fs_add_binary(fs, "udp_payload", payload_len, (void *)payload, + 0); + fs_add_null_icmp(fs); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "udp_payload"); + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + } +} + +int bacnet_global_initialize(struct state_conf *conf) +{ + num_ports = conf->source_port_last - conf->source_port_first + 1; + return EXIT_SUCCESS; +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + {.name = "udp_payload", .type = "binary", .desc = "UDP payload"}, + ICMP_FIELDSET_FIELDS, +}; + +probe_module_t module_bacnet = {.name = "bacnet", + .max_packet_length = ZMAP_BACNET_PACKET_LEN, + .pcap_filter = "udp || icmp", + .pcap_snaplen = 1500, + .port_args = 1, + .thread_initialize = &bacnet_init_perthread, + .global_initialize = &bacnet_global_initialize, + .make_packet = &bacnet_make_packet, + .print_packet = &udp_print_packet, + .validate_packet = &bacnet_validate_packet, + .process_packet = &bacnet_process_packet, + .close = &udp_global_cleanup, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = + sizeof(fields) / sizeof(fields[0])}; diff --git a/src/probe_modules/module_bacnet.h b/src/probe_modules/module_bacnet.h new file mode 100644 index 0000000..6dc4091 --- /dev/null +++ b/src/probe_modules/module_bacnet.h @@ -0,0 +1,57 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_MODULE_BACNET_H +#define ZMAP_MODULE_BACNET_H + +#include +#include +#include +#include +#include + +#include "probe_modules.h" + +extern probe_module_t module_bacnet; + +struct __attribute__((__packed__)) bacnet_vlc { + uint8_t type; + uint8_t function; + uint16_t length; +}; +typedef struct bacnet_vlc bacnet_vlc_t; + +struct __attribute__((__packed__)) bacnet_npdu { + uint8_t version; + uint8_t control; +}; +typedef struct bacnet_npdu bacnet_npdu_t; + +struct __attribute__((__packed__)) bacnet_apdu { + uint8_t type_flags; + uint8_t max_segments_apdu; + uint8_t invoke_id; + uint8_t server_choice; +}; +typedef struct bacnet_apdu bacnet_apdu_t; + +struct __attribute__((__packed__)) bacnet_probe { + struct bacnet_vlc vlc; + struct bacnet_npdu npdu; + struct bacnet_apdu apdu; +}; +typedef struct bacnet_probe bacnet_probe_t; + +typedef struct bacnet_oid bacnet_oid_t; + +#define ZMAP_BACNET_TYPE_IP 0x81 +#define ZMAP_BACNET_FUNCTION_UNICAST_NPDU 0x0a +#define ZMAP_BACNET_NPDU_VERSION_ASHRAE_135_1995 0x01 +#define ZMAP_BACNET_SERVER_CHOICE_READ_PROPERTY 0x0c + +#endif /* ZMAP_MODULE_BACNET_H */ diff --git a/src/probe_modules/module_dns.c b/src/probe_modules/module_dns.c new file mode 100644 index 0000000..8a4bdb5 --- /dev/null +++ b/src/probe_modules/module_dns.c @@ -0,0 +1,1136 @@ +/* + * ZMap Copyright 2015 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// Module for scanning for open UDP DNS resolvers. +// +// This module optionally takes in an argument of the form "TYPE,QUESTION" +// (e.g. "A,google.com"). +// +// Given no arguments it will default to asking for an A record for +// www.google.com. +// +// This module does minimal answer verification. It only verifies that the +// response roughly looks like a DNS response. It will not, for example, +// require the QR bit be set to 1. All such analysis should happen offline. +// Specifically, to be included in the output it requires: +// - That the response packet is >= the query packet. +// - That the ports match and the packet is complete. +// To be marked as success it also requires: +// - That the response bytes that should be the ID field matches the send bytes. +// - That the response bytes that should be question match send bytes. +// To be marked as app_success it also requires: +// - That the QR bit be 1 and rcode == 0. +// +// Usage: +// zmap -p 53 --probe-module=dns --probe-args="ANY,www.example.com" +// -O json --output-fields=* 8.8.8.8 +// +// We also support multiple questions, of the form: +// "A,example.com;AAAA,www.example.com" This requires --probes=X, where X +// is a multiple of the number of questions in --probe-args, and either +// --output-filter="" or --output-module=csv to remove the implicit +// "filter_duplicates" configuration flag. +// + +#include "module_dns.h" +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../../lib/random.h" +#include "../../lib/xalloc.h" +#include "probe_modules.h" +#include "packet.h" +#include "logger.h" +#include "module_udp.h" +#include "../fieldset.h" + +#define DNS_PAYLOAD_LEN_LIMIT 512 // This is arbitrary +#define PCAP_SNAPLEN 1500 // This is even more arbitrary +#define MAX_QTYPE 255 +#define ICMP_UNREACH_HEADER_SIZE 8 +#define BAD_QTYPE_STR "BAD QTYPE" +#define BAD_QTYPE_VAL -1 +#define MAX_LABEL_RECURSION 10 +#define DNS_QR_ANSWER 1 + +// Note: each label has a max length of 63 bytes. So someone has to be doing +// something really annoying. Will raise a warning. +// THIS INCLUDES THE NULL BYTE +#define MAX_NAME_LENGTH 512 + +#if defined(__NetBSD__) && !defined(__cplusplus) && defined(bool) +#undef bool +#endif + +typedef uint8_t bool; + +// zmap boilerplate +probe_module_t module_dns; +static int num_ports; + +const char default_domain[] = "www.google.com"; +const uint16_t default_qtype = DNS_QTYPE_A; +const uint8_t default_rdbit = 0xFF; + +static char **dns_packets; +static uint16_t *dns_packet_lens; // Not including udp header +static uint16_t *qname_lens; +static char **qnames; +static uint16_t *qtypes; +static int num_questions = 0; // How many DNS questions to query. Note: There's a requirement that probes is a multiple of DNS questions +// necessary to null-terminate these since strtrk_r can take multiple delimitors as a char*, and since these are contiguous in memory, +// they were being used jointly when the intention is to use only one at a time. +static const char* probe_arg_delimitor = ";\0"; +static const char* domain_qtype_delimitor = ",\0"; +static const char* rn_delimitor = ":\0"; + +static uint8_t *rdbits; +const char *qopts_rn = "nr"; // used in query to disable recursion bit in DNS header + +/* Array of qtypes we support. Jumping through some hoops (1 level of + * indirection) so the per-packet processing time is fast. Keep this in sync + * with: dns_qtype (.h) qtype_strid_to_qtype (below) qtype_qtype_to_strid + * (below, and setup_qtype_str_map()) + */ +const char *qtype_strs[] = {"A", "NS", "CNAME", "SOA", "PTR", + "MX", "TXT", "AAAA", "RRSIG", "ALL"}; +const int qtype_strs_len = 10; + +const dns_qtype qtype_strid_to_qtype[] = { + DNS_QTYPE_A, DNS_QTYPE_NS, DNS_QTYPE_CNAME, DNS_QTYPE_SOA, + DNS_QTYPE_PTR, DNS_QTYPE_MX, DNS_QTYPE_TXT, DNS_QTYPE_AAAA, + DNS_QTYPE_RRSIG, DNS_QTYPE_ALL}; + +int8_t qtype_qtype_to_strid[256] = {BAD_QTYPE_VAL}; + +void setup_qtype_str_map(void) +{ + qtype_qtype_to_strid[DNS_QTYPE_A] = 0; + qtype_qtype_to_strid[DNS_QTYPE_NS] = 1; + qtype_qtype_to_strid[DNS_QTYPE_CNAME] = 2; + qtype_qtype_to_strid[DNS_QTYPE_SOA] = 3; + qtype_qtype_to_strid[DNS_QTYPE_PTR] = 4; + qtype_qtype_to_strid[DNS_QTYPE_MX] = 5; + qtype_qtype_to_strid[DNS_QTYPE_TXT] = 6; + qtype_qtype_to_strid[DNS_QTYPE_AAAA] = 7; + qtype_qtype_to_strid[DNS_QTYPE_RRSIG] = 8; + qtype_qtype_to_strid[DNS_QTYPE_ALL] = 9; +} + +static uint16_t qtype_str_to_code(const char *str) +{ + for (int i = 0; i < qtype_strs_len; i++) { + if (strcmp(qtype_strs[i], str) == 0) + return qtype_strid_to_qtype[i]; + } + return 0; +} + +static uint16_t domain_to_qname(char **qname_handle, const char *domain) +{ + // String + 1byte header + null byte + uint16_t len = strlen(domain) + 1 + 1; + char *qname = xmalloc(len); + // Add a . before the domain. This will make the following simpler. + qname[0] = '.'; + // Move the domain into the qname buffer. + strcpy(qname + 1, domain); + for (int i = 0; i < len; i++) { + if (qname[i] == '.') { + int j; + for (j = i + 1; j < (len - 1); j++) { + if (qname[j] == '.') { + break; + } + } + qname[i] = j - i - 1; + } + } + *qname_handle = qname; + assert((*qname_handle)[len - 1] == '\0'); + return len; +} + +static int build_global_dns_packets(char *domains[], int num_domains, size_t *max_len) +{ + size_t _max_len = 0; + for (int i = 0; i < num_domains; i++) { + + qname_lens[i] = domain_to_qname(&qnames[i], domains[i]); + if (domains[i] != (char *)default_domain) { + free(domains[i]); + } + uint16_t len = sizeof(dns_header) + qname_lens[i] + + sizeof(dns_question_tail); + dns_packet_lens[i] = len; + if (len > _max_len) { + _max_len = len; + } + + if (dns_packet_lens[i] > DNS_PAYLOAD_LEN_LIMIT) { + log_fatal("dns", + "DNS packet bigger (%d) than our limit (%d)", + dns_packet_lens[i], DNS_PAYLOAD_LEN_LIMIT); + return EXIT_FAILURE; + } + + dns_packets[i] = xmalloc(dns_packet_lens[i]); + + dns_header *dns_header_p = (dns_header *)dns_packets[i]; + char *qname_p = dns_packets[i] + sizeof(dns_header); + dns_question_tail *tail_p = + (dns_question_tail *)(dns_packets[i] + sizeof(dns_header) + + qname_lens[i]); + + // All other header fields should be 0. Except id, which we set + // per thread. Please recurse as needed. + dns_header_p->rd = rdbits[i]; + + // We have 1 question + dns_header_p->qdcount = htons(1); + memcpy(qname_p, qnames[i], qname_lens[i]); + // Set the qtype to what we passed from args + tail_p->qtype = htons(qtypes[i]); + // Set the qclass to The Internet + tail_p->qclass = htons(0x01); + } + *max_len = _max_len; + return EXIT_SUCCESS; +} + +static uint16_t get_name_helper(const char *data, uint16_t data_len, + const char *payload, uint16_t payload_len, + char *name, uint16_t name_len, + uint16_t recursion_level) +{ + log_trace("dns", + "_get_name_helper IN, datalen: %d namelen: %d recursion: %d", + data_len, name_len, recursion_level); + if (data_len == 0 || name_len == 0 || payload_len == 0) { + log_trace( + "dns", + "_get_name_helper OUT, err. 0 length field. datalen %d namelen %d payloadlen %d", + data_len, name_len, payload_len); + return 0; + } + if (recursion_level > MAX_LABEL_RECURSION) { + log_trace("dns", "_get_name_helper OUT. ERR, MAX RECURSION"); + return 0; + } + uint16_t bytes_consumed = 0; + // The start of data is either a sequence of labels or a ptr. + while (data_len > 0) { + uint8_t byte = data[0]; + // Is this a pointer? + if (byte >= 0xc0) { + log_trace("dns", "_get_name_helper, ptr encountered"); + // Do we have enough bytes to check ahead? + if (data_len < 2) { + log_trace( + "dns", + "_get_name_helper OUT. ptr byte encountered. No offset. ERR."); + return 0; + } + // No. ntohs isn't needed here. It's because of + // the upper 2 bits indicating a pointer. + uint16_t offset = + ((byte & 0x03) << 8) | (uint8_t)data[1]; + log_trace("dns", "_get_name_helper. ptr offset 0x%x", + offset); + if (offset >= payload_len) { + log_trace( + "dns", + "_get_name_helper OUT. offset exceeded payload len %d ERR", + payload_len); + return 0; + } + + // We need to add a dot if we are: + // -- Not first level recursion. + // -- have consumed bytes + if (recursion_level > 0 || bytes_consumed > 0) { + + if (name_len < 1) { + log_warn( + "dns", + "Exceeded static name field allocation."); + return 0; + } + + name[0] = '.'; + name++; + name_len--; + } + uint16_t rec_bytes_consumed = get_name_helper( + payload + offset, payload_len - offset, payload, + payload_len, name, name_len, recursion_level + 1); + // We are done so don't bother to increment the + // pointers. + if (rec_bytes_consumed == 0) { + log_trace( + "dns", + "_get_name_helper OUT. rec level %d failed", + recursion_level); + return 0; + } else { + bytes_consumed += 2; + log_trace( + "dns", + "_get_name_helper OUT. rec level %d success. " + "%d rec bytes consumed. %d bytes consumed.", + recursion_level, rec_bytes_consumed, + bytes_consumed); + return bytes_consumed; + } + } else if (byte == '\0') { + // don't bother with pointer incrementation. We're done. + bytes_consumed += 1; + log_trace( + "dns", + "_get_name_helper OUT. rec level %d success. %d bytes consumed.", + recursion_level, bytes_consumed); + return bytes_consumed; + } else { + log_trace("dns", + "_get_name_helper, segment 0x%hx encountered", + byte); + // We've now consumed a byte. + ++data; + --data_len; + // Mark byte consumed after we check for first + // iteration. Do we have enough data left (must have + // null byte too)? + if ((byte + 1) > data_len) { + log_trace( + "dns", + "_get_name_helper OUT. ERR. Not enough data for segment %hd"); + return 0; + } + // If we've consumed any bytes and are in a label, we're + // in a label chain. We need to add a dot. + if (bytes_consumed > 0) { + + if (name_len < 1) { + log_warn( + "dns", + "Exceeded static name field allocation."); + return 0; + } + + name[0] = '.'; + name++; + name_len--; + } + // Now we've consumed a byte. + ++bytes_consumed; + // Did we run out of our arbitrary buffer? + if (byte > name_len) { + log_warn( + "dns", + "Exceeded static name field allocation."); + return 0; + } + + assert(data_len > 0); + memcpy(name, data, byte); + name += byte; + name_len -= byte; + data_len -= byte; + data += byte; + bytes_consumed += byte; + // Handled in the byte+1 check above. + assert(data_len > 0); + } + } + // We should never get here. + // For each byte we either have: + // -- a ptr, which terminates + // -- a null byte, which terminates + // -- a segment length which either terminates or ensures we keep + // looping + assert(0); + return 0; +} + +// data: Where we are in the dns payload +// payload: the entire udp payload +static char *get_name(const char *data, uint16_t data_len, const char *payload, + uint16_t payload_len, uint16_t *bytes_consumed) +{ + log_trace("dns", "call to get_name, data_len: %d", data_len); + char *name = xmalloc(MAX_NAME_LENGTH); + *bytes_consumed = get_name_helper(data, data_len, payload, payload_len, + name, MAX_NAME_LENGTH - 1, 0); + if (*bytes_consumed == 0) { + free(name); + return NULL; + } + // Our memset ensured null byte. + assert(name[MAX_NAME_LENGTH - 1] == '\0'); + log_trace( + "dns", + "return success from get_name, bytes_consumed: %d, string: %s", + *bytes_consumed, name); + return name; +} + +static bool process_response_question(char **data, uint16_t *data_len, + const char *payload, uint16_t payload_len, + fieldset_t *list) +{ + // Payload is the start of the DNS packet, including header + // data is handle to the start of this RR + // data_len is a pointer to the how much total data we have to work + // with. This is awful. I'm bad and should feel bad. + uint16_t bytes_consumed = 0; + char *question_name = + get_name(*data, *data_len, payload, payload_len, &bytes_consumed); + // Error. + if (question_name == NULL) { + return 1; + } + assert(bytes_consumed > 0); + if ((bytes_consumed + sizeof(dns_question_tail)) > *data_len) { + free(question_name); + return 1; + } + dns_question_tail *tail = (dns_question_tail *)(*data + bytes_consumed); + uint16_t qtype = ntohs(tail->qtype); + uint16_t qclass = ntohs(tail->qclass); + // Build our new question fieldset + fieldset_t *qfs = fs_new_fieldset(NULL); + fs_add_unsafe_string(qfs, "name", question_name, 1); + fs_add_uint64(qfs, "qtype", qtype); + if (qtype > MAX_QTYPE || qtype_qtype_to_strid[qtype] == BAD_QTYPE_VAL) { + fs_add_string(qfs, "qtype_str", (char *)BAD_QTYPE_STR, 0); + } else { + // I've written worse things than this 3rd arg. But I want to be + // fast. + fs_add_string(qfs, "qtype_str", + (char *)qtype_strs[qtype_qtype_to_strid[qtype]], + 0); + } + fs_add_uint64(qfs, "qclass", qclass); + // Now we're adding the new fs to the list. + fs_add_fieldset(list, NULL, qfs); + // Now update the pointers. + *data = *data + bytes_consumed + sizeof(dns_question_tail); + *data_len = *data_len - bytes_consumed - sizeof(dns_question_tail); + return 0; +} + +static bool process_response_answer(char **data, uint16_t *data_len, + const char *payload, uint16_t payload_len, + fieldset_t *list) +{ + log_trace("dns", "call to process_response_answer, data_len: %d", + *data_len); + // Payload is the start of the DNS packet, including header + // data is handle to the start of this RR + // data_len is a pointer to the how much total data we have to work + // with. This is awful. I'm bad and should feel bad. + uint16_t bytes_consumed = 0; + char *answer_name = + get_name(*data, *data_len, payload, payload_len, &bytes_consumed); + // Error. + if (answer_name == NULL) { + return 1; + } + assert(bytes_consumed > 0); + if ((bytes_consumed + sizeof(dns_answer_tail)) > *data_len) { + free(answer_name); + return 1; + } + dns_answer_tail *tail = (dns_answer_tail *)(*data + bytes_consumed); + uint16_t type = ntohs(tail->type); + uint16_t class = ntohs(tail->class); + uint32_t ttl = ntohl(tail->ttl); + uint16_t rdlength = ntohs(tail->rdlength); + char *rdata = tail->rdata; + + if ((rdlength + bytes_consumed + sizeof(dns_answer_tail)) > *data_len) { + free(answer_name); + return 1; + } + // Build our new question fieldset + fieldset_t *afs = fs_new_fieldset(NULL); + fs_add_unsafe_string(afs, "name", answer_name, 1); + fs_add_uint64(afs, "type", type); + if (type > MAX_QTYPE || qtype_qtype_to_strid[type] == BAD_QTYPE_VAL) { + fs_add_string(afs, "type_str", (char *)BAD_QTYPE_STR, 0); + } else { + // I've written worse things than this 3rd arg. But I want to be + // fast. + fs_add_string(afs, "type_str", + (char *)qtype_strs[qtype_qtype_to_strid[type]], + 0); + } + fs_add_uint64(afs, "class", class); + fs_add_uint64(afs, "ttl", ttl); + fs_add_uint64(afs, "rdlength", rdlength); + + // XXX Fill this out for the other types we care about. + if (type == DNS_QTYPE_NS || type == DNS_QTYPE_CNAME) { + uint16_t rdata_bytes_consumed = 0; + char *rdata_name = get_name(rdata, rdlength, payload, + payload_len, &rdata_bytes_consumed); + if (rdata_name == NULL) { + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + fs_add_uint64(afs, "rdata_is_parsed", 1); + fs_add_unsafe_string(afs, "rdata", rdata_name, 1); + } + } else if (type == DNS_QTYPE_MX) { + uint16_t rdata_bytes_consumed = 0; + if (rdlength <= 4) { + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + char *rdata_name = + get_name(rdata + 2, rdlength - 2, payload, + payload_len, &rdata_bytes_consumed); + if (rdata_name == NULL) { + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + // (largest value 16bit) + " " + answer + null + char *rdata_with_pref = + xmalloc(5 + 1 + strlen(rdata_name) + 1); + + uint8_t num_printed = + snprintf(rdata_with_pref, 6, "%hu ", + ntohs(*(uint16_t *)rdata)); + memcpy(rdata_with_pref + num_printed, + rdata_name, strlen(rdata_name)); + fs_add_uint64(afs, "rdata_is_parsed", 1); + fs_add_unsafe_string(afs, "rdata", + rdata_with_pref, 1); + } + } + } else if (type == DNS_QTYPE_TXT) { + if (rdlength >= 1 && (rdlength - 1) != *(uint8_t *)rdata) { + log_warn( + "dns", + "TXT record with wrong TXT len. Not processing."); + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + fs_add_uint64(afs, "rdata_is_parsed", 1); + char *txt = xmalloc(rdlength); + memcpy(txt, rdata + 1, rdlength - 1); + fs_add_unsafe_string(afs, "rdata", txt, 1); + } + } else if (type == DNS_QTYPE_A) { + if (rdlength != 4) { + log_warn( + "dns", + "A record with IP of length %d. Not processing.", + rdlength); + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + fs_add_uint64(afs, "rdata_is_parsed", 1); + char *addr = + strdup(inet_ntoa(*(struct in_addr *)rdata)); + fs_add_unsafe_string(afs, "rdata", addr, 1); + } + } else if (type == DNS_QTYPE_AAAA) { + if (rdlength != 16) { + log_warn( + "dns", + "AAAA record with IP of length %d. Not processing.", + rdlength); + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + fs_add_uint64(afs, "rdata_is_parsed", 1); + char *ipv6_str = xmalloc(INET6_ADDRSTRLEN); + + inet_ntop(AF_INET6, (struct sockaddr_in6 *)rdata, + ipv6_str, INET6_ADDRSTRLEN); + + fs_add_unsafe_string(afs, "rdata", ipv6_str, 1); + } + } else { + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } + // Now we're adding the new fs to the list. + fs_add_fieldset(list, NULL, afs); + // Now update the pointers. + *data = *data + bytes_consumed + sizeof(dns_answer_tail) + rdlength; + *data_len = + *data_len - bytes_consumed - sizeof(dns_answer_tail) - rdlength; + log_trace("dns", + "return success from process_response_answer, data_len: %d", + *data_len); + return 0; +} + +/* + * Start of required zmap exports. + */ + + + +static int dns_global_initialize(struct state_conf *conf) +{ + setup_qtype_str_map(); + // strip off any leading or trailing semicolons + if (*conf->probe_args == probe_arg_delimitor[0]) { + log_debug("dns", "Probe args (%s) contains leading semicolon. Stripping.", conf->probe_args); + conf->probe_args++; + } + if (conf->probe_args[strlen(conf->probe_args) - 1] == probe_arg_delimitor[0]) { + log_debug("dns", "Probe args (%s) contains trailing semicolon. Stripping.", conf->probe_args); + conf->probe_args[strlen(conf->probe_args) - 1] = '\0'; + } + + char **domains = NULL; + num_questions = 0; + + if (conf->probe_args) { + char* questions_ctx; + char* domain_ctx; + char* domain_and_qtype = strtok_r(conf->probe_args, probe_arg_delimitor, &questions_ctx); + + // Process each pair + while (domain_and_qtype != NULL) { + // resize the array to accommodate the new pair + domains = xrealloc(domains, (num_questions + 1) * sizeof(char *)); + qtypes = xrealloc(qtypes, (num_questions + 1) * sizeof(uint16_t)); + rdbits = xrealloc(rdbits, (num_questions + 1) * sizeof(uint8_t)); + rdbits[num_questions] = default_rdbit; + + // Tokenize pair based on comma + char *qtype_token = strtok_r(domain_and_qtype, domain_qtype_delimitor, &domain_ctx); + char *domain_token = strtok_r(NULL, domain_qtype_delimitor, &domain_ctx); + if (strchr(qtype_token, rn_delimitor[0]) != NULL) { + // need to check if user supplied the no-recursion bit + char* rbit_ctx; + char *recurse_token = strtok_r(qtype_token, rn_delimitor, &rbit_ctx); + recurse_token = strtok_r(NULL, rn_delimitor, &rbit_ctx); + // check if the no-recursion field matches the expected value ("nr") + if (strcmp(recurse_token, qopts_rn) == 0) { + rdbits[num_questions] = 0; + } else { + log_warn("dns", "invalid text after DNS query type (%s). no recursion set with \"nr\"", recurse_token); + } + } + if (domain_token == NULL || qtype_token == NULL) { + log_fatal( "dns", "Invalid probe args (%s). Format: \"A,google.com\" " "or \"A,google.com;A,example.com\"", conf->probe_args); + } + if (strlen(domain_token) == 0) { + log_fatal( "dns", "Invalid domain, domain cannot be empty."); + } + uint domain_len = strlen(domain_token); + // add space for the null terminator + char* domain_ptr = xmalloc(domain_len + 1); + strncpy(domain_ptr, domain_token, domain_len); + // add null terminator + domain_ptr[domain_len] = '\0'; + + + // print debug info + if (rdbits[num_questions] == 0) { + // recursion disabled + log_debug("dns", "parsed user input to scan domain (%s), for qtype (%s) w/o recursion", domain_ptr, qtype_token); + } else { + log_debug("dns", "parsed user input to scan domain (%s), for qtype (%s) with recursion", domain_ptr, qtype_token); + } + // add the new pair to the array + domains[num_questions] = domain_ptr; + qtypes[num_questions] = qtype_str_to_code(qtype_token); + + if (!qtypes[num_questions]) { + log_fatal("dns", "Incorrect qtype supplied. %s", qtype_token); + } + + // move to the next pair of domain/qtype + domain_and_qtype = strtok_r(NULL, probe_arg_delimitor, &questions_ctx); + num_questions++; + } + } + + if (num_questions == 0) { + // user didn't provide any questions, setting up a default + log_warn("dns", "no dns questions provided, using default domain (%s) and qtype (%s)", default_domain, qtype_strs[qtype_qtype_to_strid[default_qtype]]); + // Resize the array to accommodate the new pair + domains = xrealloc(domains, (num_questions + 1) * sizeof(char *)); + qtypes = xrealloc(qtypes, (num_questions + 1) * sizeof(uint16_t)); + + // Add the new pair to the array + domains[num_questions] = strdup(default_domain); + qtypes[num_questions] = default_qtype; + + num_questions = 1; + } else { + log_debug("dns", "number of dns questions: %d", num_questions); + } + + if (conf->packet_streams % num_questions != 0) { + // probe count must be a multiple of the number of DNS questions + log_fatal("dns", "number of probes (%d) must be a multiple of the number of DNS questions (%d)." + "Example: '-P 4 --probe-args \"A,google.com;AAAA,cloudflare.com\"'", conf->packet_streams, num_questions); + } + // Setup the global structures + dns_packets = xmalloc(sizeof(char *) * num_questions); + dns_packet_lens = xmalloc(sizeof(uint16_t) * num_questions); + qname_lens = xmalloc(sizeof(uint16_t) * num_questions); + qnames = xmalloc(sizeof(char *) * num_questions); + num_ports = conf->source_port_last - conf->source_port_first + 1; + + size_t max_payload_len; + int ret = build_global_dns_packets(domains, num_questions, &max_payload_len); + module_dns.max_packet_length = max_payload_len + sizeof(struct ether_header) + sizeof(struct ip) + sizeof(struct udphdr); + return ret; +} + +static int dns_global_cleanup(UNUSED struct state_conf *zconf, + UNUSED struct state_send *zsend, + UNUSED struct state_recv *zrecv) +{ + if (dns_packets) { + for (int i = 0; i < num_questions; i++) { + if (dns_packets[i]) { + free(dns_packets[i]); + } + } + free(dns_packets); + } + dns_packets = NULL; + + if (qnames) { + for (int i = 0; i < num_questions; i++) { + if (qnames[i]) { + free(qnames[i]); + } + } + free(qnames); + } + qnames = NULL; + + if (dns_packet_lens) { + free(dns_packet_lens); + } + + if (qname_lens) { + free(qname_lens); + } + + if (qtypes) { + free(qtypes); + } + + return EXIT_SUCCESS; +} + +int dns_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + + // Setup assuming num_questions == 0 + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + sizeof(struct udphdr) + + dns_packet_lens[0]); + make_ip_header(ip_header, IPPROTO_UDP, len); + + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + len = sizeof(struct udphdr) + dns_packet_lens[0]; + make_udp_header(udp_header, len); + + char *payload = (char *)(&udp_header[1]); + + memcpy(payload, dns_packets[0], dns_packet_lens[0]); + + return EXIT_SUCCESS; +} + +// get_dns_question_index_by_probe_num - Find the dns question associated with this probe number +// We allow users to enter a probe count that is a multiple of the number of DNS questions. +// send.c will iterate with this probe count, sending a packet for each probe number +// Ex. -P 4 --probe-args="A,google.com;AAAA,cloudflare.com" - send 2 probes for each question +// Probe_num | num_questions = dns_index +// 0 | 2 = 0 +// 1 | 2 = 1 +// 2 | 2 = 0 +// 3 | 2 = 1 +int get_dns_question_index_by_probe_num(int probe_num) { + assert(probe_num >= 0); + return probe_num % num_questions; +} + +int dns_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, port_n_t dport, uint8_t ttl, + uint32_t *validation, int probe_num, UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)&ip_header[1]; + + // For num_questions == 0, we handle this in per-thread init. Do less + // work + if (num_questions > 0) { + int dns_index = get_dns_question_index_by_probe_num(probe_num); + uint16_t encoded_len = + htons(sizeof(struct ip) + sizeof(struct udphdr) + + dns_packet_lens[dns_index]); + make_ip_header(ip_header, IPPROTO_UDP, encoded_len); + + encoded_len = + sizeof(struct udphdr) + dns_packet_lens[dns_index]; + make_udp_header(udp_header, encoded_len); + + char *payload = (char *)(&udp_header[1]); + *buf_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr) + dns_packet_lens[dns_index]; + + assert(*buf_len <= MAX_PACKET_SIZE); + + memcpy(payload, dns_packets[dns_index], + dns_packet_lens[dns_index]); + } + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + // Above we wanted to look up the dns question index (so we could send 2 probes for the same DNS query) + // Here we want the port to be unique regardless of if this is the 2nd probe to the same DNS query so using + // probe_num itself to set the unique UDP source port. + udp_header->uh_sport = + htons(get_src_port(num_ports, probe_num, validation)); + udp_header->uh_dport = dport; + + dns_header *dns_header_p = (dns_header *)&udp_header[1]; + + dns_header_p->id = validation[2] & 0xFFFF; + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + return EXIT_SUCCESS; +} + +void dns_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct udphdr *udph = (struct udphdr *)(&iph[1]); + fprintf(fp, PRINT_PACKET_SEP); + fprintf(fp, "dns { source: %u | dest: %u | checksum: %#04X }\n", + ntohs(udph->uh_sport), ntohs(udph->uh_dport), + ntohs(udph->uh_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +int dns_validate_packet(const struct ip *ip_hdr, uint32_t len, uint32_t *src_ip, + uint32_t *validation, const struct port_conf *ports) +{ + // this does the heavy lifting including ICMP validation + if (udp_do_validate_packet(ip_hdr, len, src_ip, validation, num_ports, + SRC_PORT_VALIDATION, + ports) == PACKET_INVALID) { + return PACKET_INVALID; + } + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + if (!udp) { + return PACKET_INVALID; + } + // verify our packet length + uint16_t udp_len = ntohs(udp->uh_ulen); + int match = 0; + for (int i = 0; i < num_questions; i++) { + if (udp_len >= dns_packet_lens[i]) { + match += 1; + } + } + if (match == 0) { + return PACKET_INVALID; + } + if (len < udp_len) { + return PACKET_INVALID; + } + } + return PACKET_VALID; +} + +void dns_add_null_fs(fieldset_t *fs) +{ + fs_add_null(fs, "dns_id"); + fs_add_null(fs, "dns_rd"); + fs_add_null(fs, "dns_tc"); + fs_add_null(fs, "dns_aa"); + fs_add_null(fs, "dns_opcode"); + fs_add_null(fs, "dns_qr"); + fs_add_null(fs, "dns_rcode"); + fs_add_null(fs, "dns_cd"); + fs_add_null(fs, "dns_ad"); + fs_add_null(fs, "dns_z"); + fs_add_null(fs, "dns_ra"); + fs_add_null(fs, "dns_qdcount"); + fs_add_null(fs, "dns_ancount"); + fs_add_null(fs, "dns_nscount"); + fs_add_null(fs, "dns_arcount"); + + fs_add_repeated(fs, "dns_questions", fs_new_repeated_fieldset()); + fs_add_repeated(fs, "dns_answers", fs_new_repeated_fieldset()); + fs_add_repeated(fs, "dns_authorities", fs_new_repeated_fieldset()); + fs_add_repeated(fs, "dns_additionals", fs_new_repeated_fieldset()); + + fs_add_uint64(fs, "dns_parse_err", 1); + fs_add_uint64(fs, "dns_unconsumed_bytes", 0); +} + +void dns_process_packet(const u_char *packet, uint32_t len, fieldset_t *fs, + uint32_t *validation, + __attribute__((unused)) struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp_hdr = get_udp_header(ip_hdr, len); + assert(udp_hdr); + uint16_t udp_len = ntohs(udp_hdr->uh_ulen); + + int match = 0; + bool is_valid = 0; + for (int i = 0; i < num_questions; i++) { + if (udp_len < dns_packet_lens[i]) { + continue; + } + match += 1; + + char *qname_p = NULL; + dns_question_tail *tail_p = NULL; + dns_header *dns_header_p = (dns_header *)&udp_hdr[1]; + // verify our dns transaction id + if (dns_header_p->id == (validation[2] & 0xFFFF)) { + // Verify our question + qname_p = + (char *)dns_header_p + sizeof(dns_header); + tail_p = + (dns_question_tail *)(dns_packets[i] + + sizeof(dns_header) + + qname_lens[i]); + // Verify our qname + if (strcmp(qnames[i], qname_p) == 0) { + // Verify the qtype and qclass. + if (tail_p->qtype == htons(qtypes[i]) && + tail_p->qclass == htons(0x01)) { + is_valid = 1; + break; + } + } + } + } + assert(match > 0); + + dns_header *dns_hdr = (dns_header *)&udp_hdr[1]; + uint16_t qr = dns_hdr->qr; + uint16_t rcode = dns_hdr->rcode; + // Success: Has the right validation bits and the right Q + // App success: has qr and rcode bits right + // Any app level parsing issues: dns_parse_err + // + fs_add_uint64(fs, "sport", ntohs(udp_hdr->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp_hdr->uh_dport)); + + // High level info + fs_add_string(fs, "classification", (char *)"dns", 0); + fs_add_bool(fs, "success", is_valid); + // additional UDP information + fs_add_bool(fs, "app_success", + is_valid && (qr == DNS_QR_ANSWER) && + (rcode == DNS_RCODE_NOERR)); + // ICMP info + fs_add_null_icmp(fs); + fs_add_uint64(fs, "udp_len", udp_len); + // DNS data + if (!is_valid) { + dns_add_null_fs(fs); + } else { + // DNS header + fs_add_uint64(fs, "dns_id", ntohs(dns_hdr->id)); + fs_add_uint64(fs, "dns_rd", dns_hdr->rd); + fs_add_uint64(fs, "dns_tc", dns_hdr->tc); + fs_add_uint64(fs, "dns_aa", dns_hdr->aa); + fs_add_uint64(fs, "dns_opcode", dns_hdr->opcode); + fs_add_uint64(fs, "dns_qr", qr); + fs_add_uint64(fs, "dns_rcode", rcode); + fs_add_uint64(fs, "dns_cd", dns_hdr->cd); + fs_add_uint64(fs, "dns_ad", dns_hdr->ad); + fs_add_uint64(fs, "dns_z", dns_hdr->z); + fs_add_uint64(fs, "dns_ra", dns_hdr->ra); + fs_add_uint64(fs, "dns_qdcount", + ntohs(dns_hdr->qdcount)); + fs_add_uint64(fs, "dns_ancount", + ntohs(dns_hdr->ancount)); + fs_add_uint64(fs, "dns_nscount", + ntohs(dns_hdr->nscount)); + fs_add_uint64(fs, "dns_arcount", + ntohs(dns_hdr->arcount)); + // And now for the complicated part. Hierarchical data. + char *data = ((char *)dns_hdr) + sizeof(dns_header); + uint16_t data_len = + udp_len - sizeof(udp_hdr) - sizeof(dns_header); + bool err = 0; + // Questions + fieldset_t *list = fs_new_repeated_fieldset(); + for (int i = 0; i < ntohs(dns_hdr->qdcount) && !err; + i++) { + err = process_response_question( + &data, &data_len, (char *)dns_hdr, udp_len, + list); + } + fs_add_repeated(fs, "dns_questions", list); + // Answers + list = fs_new_repeated_fieldset(); + for (int i = 0; i < ntohs(dns_hdr->ancount) && !err; + i++) { + err = process_response_answer(&data, &data_len, + (char *)dns_hdr, + udp_len, list); + } + fs_add_repeated(fs, "dns_answers", list); + // Authorities + list = fs_new_repeated_fieldset(); + for (int i = 0; i < ntohs(dns_hdr->nscount) && !err; + i++) { + err = process_response_answer(&data, &data_len, + (char *)dns_hdr, + udp_len, list); + } + fs_add_repeated(fs, "dns_authorities", list); + // Additionals + list = fs_new_repeated_fieldset(); + for (int i = 0; i < ntohs(dns_hdr->arcount) && !err; + i++) { + err = process_response_answer(&data, &data_len, + (char *)dns_hdr, + udp_len, list); + } + fs_add_repeated(fs, "dns_additionals", list); + // Do we have unconsumed data? + if (data_len != 0) { + err = 1; + } + // Did we parse OK? + fs_add_uint64(fs, "dns_parse_err", err); + fs_add_uint64(fs, "dns_unconsumed_bytes", data_len); + } + // Now the raw stuff. + fs_add_binary(fs, "raw_data", (udp_len - sizeof(struct udphdr)), + (void *)&udp_hdr[1], 0); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + fs_add_bool(fs, "app_success", 0); + // Populate all ICMP Fields + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + fs_add_null(fs, "udp_len"); + dns_add_null_fs(fs); + fs_add_binary(fs, "raw_data", len, (char *)packet, 0); + } else { + // This should not happen. Both the pcap filter and validate + // packet prevent this. + log_fatal("dns", "Die. This can only happen if you " + "change the pcap filter and don't update the " + "process function."); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + {.name = "app_success", + .type = "bool", + .desc = "Is the RA bit set with no error code?"}, + ICMP_FIELDSET_FIELDS, + {.name = "udp_len", .type = "int", .desc = "UDP packet length"}, + {.name = "dns_id", .type = "int", .desc = "DNS transaction ID"}, + {.name = "dns_rd", .type = "int", .desc = "DNS recursion desired"}, + {.name = "dns_tc", .type = "int", .desc = "DNS packet truncated"}, + {.name = "dns_aa", .type = "int", .desc = "DNS authoritative answer"}, + {.name = "dns_opcode", .type = "int", .desc = "DNS opcode (query type)"}, + {.name = "dns_qr", .type = "int", .desc = "DNS query(0) or response (1)"}, + {.name = "dns_rcode", .type = "int", .desc = "DNS response code"}, + {.name = "dns_cd", .type = "int", .desc = "DNS checking disabled"}, + {.name = "dns_ad", .type = "int", .desc = "DNS authenticated data"}, + {.name = "dns_z", .type = "int", .desc = "DNS reserved"}, + {.name = "dns_ra", .type = "int", .desc = "DNS recursion available"}, + {.name = "dns_qdcount", .type = "int", .desc = "DNS number questions"}, + {.name = "dns_ancount", .type = "int", .desc = "DNS number answer RR's"}, + {.name = "dns_nscount", + .type = "int", + .desc = "DNS number NS RR's in authority section"}, + {.name = "dns_arcount", + .type = "int", + .desc = "DNS number additional RR's"}, + {.name = "dns_questions", .type = "repeated", .desc = "DNS question list"}, + {.name = "dns_answers", .type = "repeated", .desc = "DNS answer list"}, + {.name = "dns_authorities", + .type = "repeated", + .desc = "DNS authority list"}, + {.name = "dns_additionals", + .type = "repeated", + .desc = "DNS additional list"}, + {.name = "dns_parse_err", + .type = "int", + .desc = "Problem parsing the DNS response"}, + {.name = "dns_unconsumed_bytes", + .type = "int", + .desc = "Bytes left over when parsing" + " the DNS response"}, + {.name = "raw_data", .type = "binary", .desc = "UDP payload"}, +}; + +probe_module_t module_dns = { + .name = "dns", + .max_packet_length = 0, // set in init + .pcap_filter = "udp || icmp", + .pcap_snaplen = PCAP_SNAPLEN, + .port_args = 1, + .thread_initialize = &dns_init_perthread, + .global_initialize = &dns_global_initialize, + .make_packet = &dns_make_packet, + .print_packet = &dns_print_packet, + .validate_packet = &dns_validate_packet, + .process_packet = &dns_process_packet, + .close = &dns_global_cleanup, + .output_type = OUTPUT_TYPE_DYNAMIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = + "This module sends out DNS queries and parses basic responses. " + "By default, the module will perform an A record lookup for " + "google.com. You can specify other queries using the --probe-args " + "argument in the form: 'type,query', e.g. 'A,google.com'. The --probes/-P " + "flag must be set to a multiple of the number of DNS questions. The module " + "supports sending the the following types of queries: A, NS, CNAME, SOA, " + "PTR, MX, TXT, AAAA, RRSIG, and ALL. In order to send queries with the " + "'recursion desired' bit set to 0, append the suffix ':nr' to the query " + "type, e.g. 'A:nr,google.com'. The module will accept and attempt " + "to parse all DNS responses. There is currently support for parsing out " + "full data from A, NS, CNAME, MX, TXT, and AAAA. Any other types will be " + "output in raw form." + +}; diff --git a/src/probe_modules/module_dns.h b/src/probe_modules/module_dns.h new file mode 100644 index 0000000..efd0679 --- /dev/null +++ b/src/probe_modules/module_dns.h @@ -0,0 +1,69 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include + +typedef struct __attribute__((packed)) { + uint16_t id; /* transaction ID */ + unsigned rd : 1; /* recursion desired */ + unsigned tc : 1; /* truncation */ + unsigned aa : 1; /* authoritative answer */ + unsigned opcode : 4; /* opcode 0=std query 1=Inverse query 2=srv status + request */ + unsigned qr : 1; /* query/response */ + + unsigned rcode : 4; /* response code */ + unsigned cd : 1; /* checking disabled */ + unsigned ad : 1; /* authenticated data */ + unsigned z : 1; /* reserved set to 0 */ + unsigned ra : 1; /* recursion available */ + + uint16_t qdcount; /* # entries in question section */ + uint16_t ancount; /* # RR in answer section */ + uint16_t nscount; /* # name server RR in authority section */ + uint16_t arcount; /* # RR in additional information section */ +} dns_header; + +typedef struct __attribute__((packed)) { + uint16_t qtype; + uint16_t qclass; +} dns_question_tail; + +typedef struct __attribute__((packed)) { + uint16_t type; + uint16_t class; + uint32_t ttl; + uint16_t rdlength; + char rdata[]; +} dns_answer_tail; + +typedef enum { + DNS_QTYPE_A = 1, + DNS_QTYPE_NS = 2, + DNS_QTYPE_CNAME = 5, + DNS_QTYPE_SOA = 6, + DNS_QTYPE_PTR = 12, + DNS_QTYPE_MX = 15, + DNS_QTYPE_TXT = 16, + DNS_QTYPE_AAAA = 28, + DNS_QTYPE_RRSIG = 46, + DNS_QTYPE_ALL = 255 +} dns_qtype; + +typedef enum { + DNS_RCODE_NOERR = 0, + DNS_RCODE_FORMATERR = 1, + DNS_RCODE_SRVFAILURE = 2, + DNS_RCODE_NXDOMAIN = 3, + DNS_RCODE_QTYPENOTIMPL = 4, + DNS_RCODE_QRYREFUSED = 5 +} dns_rcode; diff --git a/src/probe_modules/module_icmp_echo.c b/src/probe_modules/module_icmp_echo.c new file mode 100644 index 0000000..6e68b25 --- /dev/null +++ b/src/probe_modules/module_icmp_echo.c @@ -0,0 +1,345 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing ICMP echo request (ping) scans + +#include +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" +#include "probe_modules.h" +#include "../fieldset.h" +#include "packet.h" +#include "logger.h" +#include "validate.h" + +#define ICMP_SMALLEST_SIZE 5 +#define ICMP_MAX_PAYLOAD_LEN 1458 +#define ICMP_TIMXCEED_UNREACH_HEADER_SIZE 8 + +probe_module_t module_icmp_echo; + +const char *icmp_usage_error = + "unknown ICMP probe specification (expected file:/path or text:STRING or hex:01020304)"; + +static size_t icmp_payload_len = 0; +static const size_t icmp_payload_default_len = 20; +static char *icmp_payload = NULL; + +int icmp_global_initialize(struct state_conf *conf) +{ + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) { + icmp_payload = xmalloc(icmp_payload_default_len); + icmp_payload_len = icmp_payload_default_len; + return EXIT_SUCCESS; + } + + char *c = strchr(conf->probe_args, ':'); + if (!c) { + log_error("icmp", icmp_usage_error); + return EXIT_FAILURE; + } + ++c; + + if (strncmp(conf->probe_args, "text", 4) == 0) { + icmp_payload = strdup(c); + icmp_payload_len = strlen(icmp_payload); + } else if (strncmp(conf->probe_args, "file", 4) == 0) { + FILE *inp = fopen(c, "rb"); + if (!inp) { + log_error("icmp", "could not open ICMP data file '%s'", + c); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_END)) { + log_error("icmp", + "unable to get size of ICMP data file '%s'", + c); + return EXIT_FAILURE; + } + size_t input_size = ftell(inp); + if (input_size > ICMP_MAX_PAYLOAD_LEN) { + log_error( + "icmp", + "input file larger than %d bytes and will not fit on the wire (%llu bytes provided)", + ICMP_MAX_PAYLOAD_LEN, input_size); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_SET)) { + log_error("icmp", "unable to read ICMP data file '%s'", + c); + return EXIT_FAILURE; + } + icmp_payload = xmalloc(ICMP_MAX_PAYLOAD_LEN); + icmp_payload_len = + fread(icmp_payload, 1, ICMP_MAX_PAYLOAD_LEN, inp); + fclose(inp); + } else if (strncmp(conf->probe_args, "hex", 3) == 0) { + if (strlen(c) % 2 != 0) { + log_error( + "icmp", + "invalid hex input (length must be a multiple of 2)"); + return EXIT_FAILURE; + } + icmp_payload_len = strlen(c) / 2; + icmp_payload = xmalloc(icmp_payload_len); + + unsigned int n; + for (size_t i = 0; i < icmp_payload_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + free(icmp_payload); + log_error("icmp", "non-hex character: '%c'", + c[i * 2]); + return EXIT_FAILURE; + } + icmp_payload[i] = (char)(n & 0xff); + } + } else { + log_error("icmp", icmp_usage_error); + return EXIT_FAILURE; + } + + if (icmp_payload_len > ICMP_MAX_PAYLOAD_LEN) { + log_error( + "icmp", + "reducing ICMP payload must be at most %d bytes to fit on the wire (%d were provided)\n", + ICMP_MAX_PAYLOAD_LEN, icmp_payload_len); + return EXIT_FAILURE; + } + + module_icmp_echo.max_packet_length = sizeof(struct ether_header) + + sizeof(struct ip) + ICMP_MINLEN + + icmp_payload_len; + assert(module_icmp_echo.max_packet_length <= 1500); + return EXIT_SUCCESS; +} + +int icmp_global_cleanup(__attribute__((unused)) struct state_conf *zconf, + __attribute__((unused)) struct state_send *zsend, + __attribute__((unused)) struct state_recv *zrecv) +{ + if (icmp_payload) { + free(icmp_payload); + icmp_payload = NULL; + } + + return EXIT_SUCCESS; +} + +static int icmp_echo_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = + htons(sizeof(struct ip) + ICMP_MINLEN + icmp_payload_len); + make_ip_header(ip_header, IPPROTO_ICMP, len); + + struct icmp *icmp_header = (struct icmp *)(&ip_header[1]); + make_icmp_header(icmp_header); + + char *payload = (char *)icmp_header + ICMP_MINLEN; + + memcpy(payload, icmp_payload, icmp_payload_len); + + return EXIT_SUCCESS; +} + +static int icmp_echo_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, UNUSED port_n_t dst_port, + uint8_t ttl, uint32_t *validation, + UNUSED int probe_num, UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct icmp *icmp_header = (struct icmp *)(&ip_header[1]); + + uint16_t icmp_idnum = validation[1] & 0xFFFF; + uint16_t icmp_seqnum = validation[2] & 0xFFFF; + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + icmp_header->icmp_id = icmp_idnum; + icmp_header->icmp_seq = icmp_seqnum; + + icmp_header->icmp_cksum = 0; + icmp_header->icmp_cksum = icmp_checksum((unsigned short *)icmp_header, + ICMP_MINLEN + icmp_payload_len); + + // Update the IP and UDP headers to match the new payload length + size_t ip_len = sizeof(struct ip) + ICMP_MINLEN + icmp_payload_len; + ip_header->ip_len = htons(ip_len); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + *buf_len = ip_len + sizeof(struct ether_header); + + return EXIT_SUCCESS; +} + +static void icmp_echo_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct icmp *icmp_header = (struct icmp *)(&iph[1]); + + fprintf(fp, + "icmp { type: %u | code: %u " + "| checksum: %#04X | id: %u | seq: %u }\n", + icmp_header->icmp_type, icmp_header->icmp_code, + ntohs(icmp_header->icmp_cksum), ntohs(icmp_header->icmp_id), + ntohs(icmp_header->icmp_seq)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +static int imcp_validate_id_seq(struct icmp *icmp_h, uint32_t *validation) +{ + if (icmp_h->icmp_id != (validation[1] & 0xFFFF)) { + return PACKET_INVALID; + } + if (icmp_h->icmp_seq != (validation[2] & 0xFFFF)) { + return PACKET_INVALID; + } + return PACKET_VALID; +} + +static int icmp_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED uint32_t *src_ip, uint32_t *validation, + UNUSED const struct port_conf *ports) +{ + if (ip_hdr->ip_p != IPPROTO_ICMP) { + return PACKET_INVALID; + } + struct icmp *icmp_h = get_icmp_header(ip_hdr, len); + if (!icmp_h) { + return PACKET_INVALID; + } + if (icmp_h->icmp_type == ICMP_ECHOREPLY) { + return imcp_validate_id_seq(icmp_h, validation); + } else { + // handle unresearch/quench/redirect/timeout + struct ip *ip_inner; + size_t ip_inner_len; + int icmp_inner_valid = icmp_helper_validate( + ip_hdr, len, sizeof(struct icmp), &ip_inner, &ip_inner_len); + if (icmp_inner_valid == PACKET_INVALID) { + return PACKET_INVALID; + } + struct icmp *icmp_inner = + get_icmp_header(ip_inner, ip_inner_len); + if (!icmp_inner) { + return PACKET_INVALID; + } + validate_gen(ip_hdr->ip_dst.s_addr, ip_inner->ip_dst.s_addr, 0, + (uint8_t *)validation); + // validate icmp id and seqnum + return imcp_validate_id_seq(icmp_inner, validation); + } +} + +static void icmp_echo_process_packet(const u_char *packet, uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + struct icmp *icmp_hdr = + (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + fs_add_uint64(fs, "type", icmp_hdr->icmp_type); + fs_add_uint64(fs, "code", icmp_hdr->icmp_code); + fs_add_uint64(fs, "icmp_id", ntohs(icmp_hdr->icmp_id)); + fs_add_uint64(fs, "seq", ntohs(icmp_hdr->icmp_seq)); + + uint32_t hdrlen = sizeof(struct ether_header) + 4 * ip_hdr->ip_hl + 4; + + switch (icmp_hdr->icmp_type) { + case ICMP_ECHOREPLY: + fs_add_string(fs, "classification", (char *)"echoreply", 0); + fs_add_uint64(fs, "success", 1); + break; + case ICMP_UNREACH: + fs_add_string(fs, "classification", (char *)"unreach", 0); + fs_add_bool(fs, "success", 0); + break; + case ICMP_SOURCEQUENCH: + fs_add_string(fs, "classification", (char *)"sourcequench", 0); + fs_add_bool(fs, "success", 0); + break; + case ICMP_REDIRECT: + fs_add_string(fs, "classification", (char *)"redirect", 0); + fs_add_bool(fs, "success", 0); + break; + case ICMP_TIMXCEED: + fs_add_string(fs, "classification", (char *)"timxceed", 0); + fs_add_bool(fs, "success", 0); + break; + default: + fs_add_string(fs, "classification", (char *)"other", 0); + fs_add_bool(fs, "success", 0); + break; + } + + int datalen = len - hdrlen; + + if (datalen > 0) { + const uint8_t *data = (uint8_t *)&packet[hdrlen]; + fs_add_binary(fs, "data", (size_t)datalen, (void *)data, 0); + } else { + fs_add_null(fs, "data"); + } +} + +static fielddef_t fields[] = { + {.name = "type", .type = "int", .desc = "icmp message type"}, + {.name = "code", .type = "int", .desc = "icmp message sub type code"}, + {.name = "icmp_id", .type = "int", .desc = "icmp id number"}, + {.name = "seq", .type = "int", .desc = "icmp sequence number"}, + {.name = "classification", + .type = "string", + .desc = "probe module classification"}, + {.name = "success", + .type = "bool", + .desc = "did probe module classify response as success"}, + {.name = "data", .type = "binary", .desc = "ICMP payload"}}; + +probe_module_t module_icmp_echo = { + .name = "icmp_echoscan", + .max_packet_length = 48, + .pcap_filter = "icmp and icmp[0]!=8", + .pcap_snaplen = 96, + .port_args = 0, + .global_initialize = &icmp_global_initialize, + .close = &icmp_global_cleanup, + .thread_initialize = &icmp_echo_init_perthread, + .make_packet = &icmp_echo_make_packet, + .print_packet = &icmp_echo_print_packet, + .process_packet = &icmp_echo_process_packet, + .validate_packet = &icmp_validate_packet, + .helptext = + "Probe module that sends ICMP echo requests to hosts.\n" + "Payload of ICMP packets will consist of zeroes unless you customize it with\n" + " --probe-args=file:/path_to_payload_file\n" + " --probe-args=text:SomeText\n" + " --probe-args=hex:5061796c6f6164", + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = 7}; diff --git a/src/probe_modules/module_icmp_echo_time.c b/src/probe_modules/module_icmp_echo_time.c new file mode 100644 index 0000000..4664ed4 --- /dev/null +++ b/src/probe_modules/module_icmp_echo_time.c @@ -0,0 +1,268 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing ICMP echo request (ping) scans that allows +// calculation of RTT + +#include +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "probe_modules.h" +#include "../fieldset.h" +#include "packet.h" +#include "validate.h" + +#define ICMP_SMALLEST_SIZE 5 +#define ICMP_TIMXCEED_UNREACH_HEADER_SIZE 8 + +probe_module_t module_icmp_echo_time; + +struct icmp_payload_for_rtt { + uint32_t sent_tv_sec; + uint32_t sent_tv_usec; + ipaddr_n_t dst; +}; + +static int icmp_echo_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + sizeof(struct icmp) - 8); + make_ip_header(ip_header, IPPROTO_ICMP, len); + + struct icmp *icmp_header = (struct icmp *)(&ip_header[1]); + make_icmp_header(icmp_header); + + return EXIT_SUCCESS; +} + +static int icmp_echo_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, UNUSED port_n_t dport, + uint8_t ttl, uint32_t *validation, + UNUSED int probe_num, UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct icmp *icmp_header = (struct icmp *)(&ip_header[1]); + struct icmp_payload_for_rtt *payload = + (struct icmp_payload_for_rtt *)(((char *)icmp_header) + 8); + + uint16_t icmp_idnum = validation[1] & 0xFFFF; + uint16_t icmp_seqnum = validation[2] & 0xFFFF; + struct timeval tv; + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + icmp_header->icmp_id = icmp_idnum; + icmp_header->icmp_seq = icmp_seqnum; + + gettimeofday(&tv, NULL); + payload->sent_tv_sec = tv.tv_sec; + payload->sent_tv_usec = tv.tv_usec; + payload->dst = dst_ip; + + icmp_header->icmp_cksum = 0; + icmp_header->icmp_cksum = + icmp_checksum((unsigned short *)icmp_header, sizeof(struct icmp)); + + // Update the IP and UDP headers to match the new payload length + size_t ip_len = sizeof(struct ip) + ICMP_MINLEN + + sizeof(struct icmp_payload_for_rtt); + ip_header->ip_len = htons(ip_len); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + *buf_len = ip_len + sizeof(struct ether_header); + return EXIT_SUCCESS; +} + +static void icmp_echo_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct icmp *icmp_header = (struct icmp *)(&iph[1]); + + fprintf(fp, + "icmp { type: %u | code: %u " + "| checksum: %#04X | id: %u | seq: %u }\n", + icmp_header->icmp_type, icmp_header->icmp_code, + ntohs(icmp_header->icmp_cksum), ntohs(icmp_header->icmp_id), + ntohs(icmp_header->icmp_seq)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +static int icmp_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation, + UNUSED const struct port_conf *ports) +{ + if (ip_hdr->ip_p != IPPROTO_ICMP) { + return 0; + } + if (((uint32_t)4 * ip_hdr->ip_hl + ICMP_SMALLEST_SIZE) > len) { + // buffer not large enough to contain expected icmp header + return 0; + } + struct icmp *icmp_h = + (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + uint16_t icmp_idnum = icmp_h->icmp_id; + uint16_t icmp_seqnum = icmp_h->icmp_seq; + // ICMP validation is tricky: for some packet types, we must look inside + // the payload + if (icmp_h->icmp_type == ICMP_TIMXCEED || + icmp_h->icmp_type == ICMP_UNREACH) { + // Should have 16B TimeExceeded/Dest_Unreachable header + + // original IP header + 1st 8B of original ICMP frame + if ((4 * ip_hdr->ip_hl + ICMP_TIMXCEED_UNREACH_HEADER_SIZE + + sizeof(struct ip)) > len) { + return 0; + } + struct ip *ip_inner = (struct ip *)((char *)icmp_h + 8); + if (((uint32_t)4 * ip_hdr->ip_hl + + ICMP_TIMXCEED_UNREACH_HEADER_SIZE + 4 * ip_inner->ip_hl + + 8 /*1st 8 bytes of original*/) > len) { + return 0; + } + struct icmp *icmp_inner = + (struct icmp *)((char *)ip_inner + 4 * ip_hdr->ip_hl); + // Regenerate validation and icmp id based off inner payload + icmp_idnum = icmp_inner->icmp_id; + icmp_seqnum = icmp_inner->icmp_seq; + *src_ip = ip_inner->ip_dst.s_addr; + validate_gen(ip_hdr->ip_dst.s_addr, ip_inner->ip_dst.s_addr, 0, + (uint8_t *)validation); + } + // validate icmp id and seqnum + if (icmp_idnum != (validation[1] & 0xFFFF)) { + return 0; + } + if (icmp_seqnum != (validation[2] & 0xFFFF)) { + return 0; + } + + return 1; +} + +static void icmp_echo_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + struct icmp *icmp_hdr = + (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + fs_add_uint64(fs, "type", icmp_hdr->icmp_type); + fs_add_uint64(fs, "code", icmp_hdr->icmp_code); + fs_add_uint64(fs, "icmp_id", ntohs(icmp_hdr->icmp_id)); + fs_add_uint64(fs, "seq", ntohs(icmp_hdr->icmp_seq)); + + struct icmp_payload_for_rtt *payload = + (struct icmp_payload_for_rtt *)(((char *)icmp_hdr) + 8); + + uint64_t sent_timestamp_ts = (uint64_t)payload->sent_tv_sec; + uint64_t sent_timestamp_us = (uint64_t)payload->sent_tv_usec; + uint64_t recv_timestamp_ts = (uint64_t)ts.tv_sec; + uint64_t recv_timestamp_us = (uint64_t)ts.tv_nsec / 1000; + uint64_t rtt_us = (recv_timestamp_ts * 1000000 + recv_timestamp_us) - + (sent_timestamp_ts * 1000000 + sent_timestamp_us); + + fs_add_uint64(fs, "sent_timestamp_ts", sent_timestamp_ts); + fs_add_uint64(fs, "sent_timestamp_us", sent_timestamp_us); + fs_add_uint64(fs, "recv_timestamp_ts", recv_timestamp_ts); + fs_add_uint64(fs, "recv_timestamp_us", recv_timestamp_us); + fs_add_uint64(fs, "rtt_us", rtt_us); + fs_add_uint64(fs, "dst_raw", (uint64_t)payload->dst); + + switch (icmp_hdr->icmp_type) { + case ICMP_ECHOREPLY: + fs_add_string(fs, "classification", (char *)"echoreply", 0); + fs_add_uint64(fs, "success", 1); + break; + case ICMP_UNREACH: + fs_add_string(fs, "classification", (char *)"unreach", 0); + fs_add_uint64(fs, "success", 0); + break; + case ICMP_SOURCEQUENCH: + fs_add_string(fs, "classification", (char *)"sourcequench", 0); + fs_add_uint64(fs, "success", 0); + break; + case ICMP_REDIRECT: + fs_add_string(fs, "classification", (char *)"redirect", 0); + fs_add_uint64(fs, "success", 0); + break; + case ICMP_TIMXCEED: + fs_add_string(fs, "classification", (char *)"timxceed", 0); + fs_add_uint64(fs, "success", 0); + break; + default: + fs_add_string(fs, "classification", (char *)"other", 0); + fs_add_uint64(fs, "success", 0); + break; + } +} + +static fielddef_t fields[] = { + {.name = "type", .type = "int", .desc = "icmp message type"}, + {.name = "code", .type = "int", .desc = "icmp message sub type code"}, + {.name = "icmp_id", .type = "int", .desc = "icmp id number"}, + {.name = "seq", .type = "int", .desc = "icmp sequence number"}, + {.name = "sent_timestamp_ts", + .type = "int", + .desc = "timestamp of sent probe in seconds since Epoch"}, + {.name = "sent_timestamp_us", + .type = "int", + .desc = "microsecond part of sent timestamp"}, + {.name = "recv_timestamp_ts", + .type = "int", + .desc = "timestamp of receive probe in seconds since Epoch"}, + {.name = "recv_timestamp_us", + .type = "int", + .desc = "microsecond part of receive timestamp"}, + {.name = "rtt_us", + .type = "int", + .desc = "round-trip time in microseconds"}, + {.name = "dst_raw", + .type = "int", + .desc = "raw destination IP address of sent probe"}, + {.name = "classification", + .type = "string", + .desc = "probe module classification"}, + {.name = "success", + .type = "int", + .desc = "did probe module classify response as success"}}; + +probe_module_t module_icmp_echo_time = { + .name = "icmp_echo_time", + .max_packet_length = 62, + .pcap_filter = "icmp and icmp[0]!=8", + .pcap_snaplen = 96, + .port_args = 0, + .thread_initialize = &icmp_echo_init_perthread, + .make_packet = &icmp_echo_make_packet, + .print_packet = &icmp_echo_print_packet, + .process_packet = &icmp_echo_process_packet, + .validate_packet = &icmp_validate_packet, + .close = NULL, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = 12}; diff --git a/src/probe_modules/module_ntp.c b/src/probe_modules/module_ntp.c new file mode 100644 index 0000000..a7ccc1d --- /dev/null +++ b/src/probe_modules/module_ntp.c @@ -0,0 +1,252 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "probe_modules.h" +#include "module_udp.h" +#include "module_ntp.h" +#include "packet.h" +#include "logger.h" + +#define MAX_NTP_PAYLOAD_LEN 1472 +#define ICMP_UNREACH_HEADER_SIZE 8 + +probe_module_t module_ntp; + +static int num_ports; + +int ntp_global_initialize(struct state_conf *conf) +{ + num_ports = conf->source_port_last - conf->source_port_first + 1; + return udp_global_initialize(conf); +} + +int ntp_validate_packet(const struct ip *ip_hdr, uint32_t len, uint32_t *src_ip, + uint32_t *validation, const struct port_conf *ports) +{ + return udp_do_validate_packet(ip_hdr, len, src_ip, validation, + num_ports, SRC_PORT_VALIDATION, ports); +} + +void ntp_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + uint64_t temp64; + uint8_t temp8; + uint32_t temp32; + + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = + (struct udphdr *)((char *)ip_hdr + ip_hdr->ip_hl * 4); + uint8_t *ptr = (uint8_t *)&udp[1]; + + fs_add_string(fs, "classification", (char *)"ntp", 0); + fs_add_bool(fs, "success", 1); + fs_add_uint64(fs, "sport", ntohs(udp->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp->uh_dport)); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); + + if (len > 90) { + temp8 = *((uint8_t *)ptr); + fs_add_uint64(fs, "LI_VN_MODE", temp8); + temp8 = *((uint8_t *)ptr + 1); + fs_add_uint64(fs, "stratum", temp8); + temp8 = *((uint8_t *)ptr + 2); + fs_add_uint64(fs, "poll", temp8); + temp8 = *((uint8_t *)ptr + 3); + fs_add_uint64(fs, "precision", temp8); + + temp32 = *((uint32_t *)ptr + 4); + fs_add_uint64(fs, "root_delay", temp32); + temp32 = *((uint32_t *)ptr + 8); + fs_add_uint64(fs, "root_dispersion", temp32); + temp32 = *((uint32_t *)ptr + 12); + fs_add_uint64(fs, "reference_clock_identifier", temp32); + + temp64 = *((uint64_t *)ptr + 16); + fs_add_uint64(fs, "reference_timestamp", temp64); + temp64 = *((uint64_t *)ptr + 24); + fs_add_uint64(fs, "originate_timestamp", temp64); + temp64 = *((uint64_t *)ptr + 32); + fs_add_uint64(fs, "receive_timestamp", temp64); + temp64 = *((uint64_t *)ptr + 39); + fs_add_uint64(fs, "transmit_timestamp", temp64); + } else { + fs_add_null(fs, "LI_VN_MODE"); + fs_add_null(fs, "stratum"); + fs_add_null(fs, "poll"); + fs_add_null(fs, "precision"); + fs_add_null(fs, "root_delay"); + fs_add_null(fs, "root_dispersion"); + fs_add_null(fs, "reference_clock_identifier"); + fs_add_null(fs, "reference_timestamp"); + fs_add_null(fs, "originate_timestamp"); + fs_add_null(fs, "receive_timestamp"); + fs_add_null(fs, "transmit_timestamp"); + } + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + struct icmp *icmp = + (struct icmp *)((char *)ip_hdr + ip_hdr->ip_hl * 4); + struct ip *ip_inner = + (struct ip *)((char *)icmp + ICMP_UNREACH_HEADER_SIZE); + + fs_modify_string(fs, "saddr", + make_ip_str(ip_inner->ip_dst.s_addr), 1); + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_string(fs, "icmp_responder", + make_ip_str(ip_hdr->ip_src.s_addr), 1); + fs_add_uint64(fs, "icmp_type", icmp->icmp_type); + fs_add_uint64(fs, "icmp_code", icmp->icmp_code); + fs_add_null(fs, "icmp_unreach_str"); + + fs_add_null(fs, "LI_VN_MODE"); + fs_add_null(fs, "stratum"); + fs_add_null(fs, "poll"); + fs_add_null(fs, "precision"); + fs_add_null(fs, "root_delay"); + fs_add_null(fs, "root_dispersion"); + fs_add_null(fs, "reference_clock_identifier"); + fs_add_null(fs, "reference_timestamp"); + fs_add_null(fs, "originate_timestamp"); + fs_add_null(fs, "receive_timestamp"); + fs_add_null(fs, "transmit_timestamp"); + + } else { + fs_add_constchar(fs, "classification", "other"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); + fs_add_null(fs, "LI_VN_MODE"); + fs_add_null(fs, "stratum"); + fs_add_null(fs, "poll"); + fs_add_null(fs, "precision"); + fs_add_null(fs, "root_delay"); + fs_add_null(fs, "root_dispersion"); + fs_add_null(fs, "reference_clock_identifier"); + fs_add_null(fs, "reference_timestamp"); + fs_add_null(fs, "originate_timestamp"); + fs_add_null(fs, "receive_timestamp"); + fs_add_null(fs, "transmit_timestamp"); + } +} + +int ntp_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, void **arg) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + sizeof(struct udphdr) + + sizeof(struct ntphdr)); + make_ip_header(ip_header, IPPROTO_UDP, len); + + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + struct ntphdr *ntp_header = (struct ntphdr *)(&udp_header[1]); + ntp_header->LI_VN_MODE = 227; + len = sizeof(struct udphdr) + sizeof(struct ntphdr); + + make_udp_header(udp_header, len); + + // TODO(dadrian): Should this have a payload? It was being set incorrectly. + size_t header_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr) + sizeof(struct ntphdr); + module_ntp.max_packet_length = header_len; + + uint32_t seed = aesrand_getword(zconf.aes); + aesrand_t *aes = aesrand_init_from_seed(seed); + *arg = aes; + + return EXIT_SUCCESS; +} + +void ntp_print_packet(FILE *fp, void *packet) +{ + + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct udphdr *udph = (struct udphdr *)(iph + 4 * iph->ip_hl); + struct ntphdr *ntph = (struct ntphdr *)&udph[1]; + fprintf(fp, "ntp { LI_VN_MODE: %u | stratum: %u | poll: %u }\n", + ntph->LI_VN_MODE, ntph->stratum, ntph->poll); + fprintf(fp, "udp { source: %u | dest: %u | checksum: %#04X }\n", + ntohs(udph->uh_sport), ntohs(udph->uh_dport), + ntohs(udph->uh_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +static fielddef_t fields[] = { + {.name = "classification", + .type = "string", + .desc = "packet classification"}, + {.name = "success", + .type = "bool", + .desc = "is response considered success"}, + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + ICMP_FIELDSET_FIELDS, + {.name = "LI_VN_MODE", + .type = "int", + .desc = "leap indication, version number, mode"}, + {.name = "stratum", .type = "int", .desc = "stratum"}, + {.name = "poll", .type = "int", .desc = "poll"}, + {.name = "precision", .type = "int", .desc = "precision"}, + {.name = "root_delay", .type = "int", .desc = "root delay"}, + {.name = "root_dispersion", .type = "int", .desc = "root dispersion"}, + {.name = "reference_clock_identifier", + .type = "int", + .desc = "code identifying clock reference"}, + {.name = "reference_timestamp", + .type = "int", + .desc = "local time at which local clock was last set or corrected"}, + {.name = "originate_timestamp", + .type = "int", + .desc = "local time at which request deparated client for service"}, + {.name = "receive_timestamp", + .type = "int", + .desc = "local time at which request arrvied at service host"}, + {.name = "transmit_timestamp", + .type = "int", + .desc = "local time which reply departed service host for client"}, +}; + +probe_module_t module_ntp = {.name = "ntp", + .max_packet_length = 0, // set in init + .pcap_filter = "udp || icmp", + .pcap_snaplen = 1500, + .port_args = 1, + .thread_initialize = &ntp_init_perthread, + .global_initialize = &ntp_global_initialize, + .make_packet = &udp_make_packet, + .print_packet = &ntp_print_packet, + .validate_packet = &ntp_validate_packet, + .process_packet = &ntp_process_packet, + .close = &udp_global_cleanup, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0])}; diff --git a/src/probe_modules/module_ntp.h b/src/probe_modules/module_ntp.h new file mode 100644 index 0000000..903a0f5 --- /dev/null +++ b/src/probe_modules/module_ntp.h @@ -0,0 +1,38 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "../state.h" +#include "../fieldset.h" + +#ifndef MODULE_NTP_H +#define MODULE_NTP_H + +#include +#include +#include +#include +#include + +struct __attribute__((__packed__)) ntphdr { // typedef + uint8_t LI_VN_MODE; + uint8_t stratum; + uint8_t poll; + uint8_t precision; + uint32_t root_delay; + uint32_t root_dispersion; + uint32_t ref_ID; + uint64_t reference_timestamp; + uint64_t origin_timestamp; + uint64_t receive_timestamp; + uint64_t transmit_timestamp; + // uint32_t key_ID; + // uint64_t dgst_1; + // uint64_t dgst_2; +}; + +#endif diff --git a/src/probe_modules/module_tcp_synackscan.c b/src/probe_modules/module_tcp_synackscan.c new file mode 100644 index 0000000..6d61dfa --- /dev/null +++ b/src/probe_modules/module_tcp_synackscan.c @@ -0,0 +1,225 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" +#include "validate.h" +#include "module_tcp_synscan.h" + +#define ZMAP_TCP_SYNACKSCAN_TCP_HEADER_LEN 24 +#define ZMAP_TCP_SYNACKSCAN_PACKET_LEN 58 + +probe_module_t module_tcp_synackscan; +static uint32_t num_ports; + +static int synackscan_global_initialize(struct state_conf *state) +{ + num_ports = state->source_port_last - state->source_port_first + 1; + return EXIT_SUCCESS; +} + +static int synackscan_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = + htons(sizeof(struct ip) + ZMAP_TCP_SYNACKSCAN_TCP_HEADER_LEN); + make_ip_header(ip_header, IPPROTO_TCP, len); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + make_tcp_header(tcp_header, TH_SYN | TH_ACK); + set_mss_option(tcp_header); + return EXIT_SUCCESS; +} + +static int synackscan_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, + port_n_t dport, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + uint32_t tcp_seq = validation[0]; + uint32_t tcp_ack = + validation[2]; // get_src_port() below uses validation 1 internally. + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + tcp_header->th_sport = + htons(get_src_port(num_ports, probe_num, validation)); + tcp_header->th_dport = dport; + tcp_header->th_seq = tcp_seq; + tcp_header->th_ack = tcp_ack; + tcp_header->th_sum = 0; + tcp_header->th_sum = tcp_checksum(ZMAP_TCP_SYNACKSCAN_TCP_HEADER_LEN, + ip_header->ip_src.s_addr, + ip_header->ip_dst.s_addr, tcp_header); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + *buf_len = ZMAP_TCP_SYNACKSCAN_PACKET_LEN; + + return EXIT_SUCCESS; +} + +static int synackscan_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED uint32_t *src_ip, + uint32_t *validation, + const struct port_conf *ports) +{ + + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len); + if (!tcp) { + return PACKET_INVALID; + } + uint16_t sport = ntohs(tcp->th_sport); + uint16_t dport = ntohs(tcp->th_dport); + // validate source port + if (!check_src_port(sport, ports)) { + return PACKET_INVALID; + } + // validate destination port + if (!check_dst_port(dport, num_ports, validation)) { + return PACKET_INVALID; + } + // check whether we'll ever send to this IP during the scan + if (!blocklist_is_allowed(*src_ip)) { + return PACKET_INVALID; + } + // We handle RST packets different than all other packets + if (tcp->th_flags & TH_RST) { + // A RST packet must have either: + // 1) resp(ack) == sent(seq) + 1, or + // 2) resp(seq) == sent(ack), or + // 3) resp(seq) == sent(ack) + 1 + // All other cases are a failure. + if (htonl(tcp->th_ack) != htonl(validation[0]) + 1 && + htonl(tcp->th_seq) != htonl(validation[2]) && + htonl(tcp->th_seq) != (htonl(validation[2]) + 1)) { + return PACKET_INVALID; + } + } else { + // For non RST packets, we must have resp(ack) == sent(seq) + 1 + if (htonl(tcp->th_ack) != htonl(validation[0]) + 1) { + return PACKET_INVALID; + } + } + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + struct ip *ip_inner; + size_t ip_inner_len; + if (icmp_helper_validate(ip_hdr, len, sizeof(struct udphdr), + &ip_inner, + &ip_inner_len) == PACKET_INVALID) { + return PACKET_INVALID; + } + struct tcphdr *tcp = get_tcp_header(ip_inner, ip_inner_len); + if (!tcp) { + return PACKET_INVALID; + } + // we can always check the destination port because this is the + // original packet and wouldn't have been altered by something + // responding on a different port + uint16_t sport = ntohs(tcp->th_sport); + uint16_t dport = ntohs(tcp->th_dport); + if (!check_src_port(dport, ports)) { + return PACKET_INVALID; + } + validate_gen(ip_hdr->ip_dst.s_addr, ip_inner->ip_dst.s_addr, + tcp->th_dport, (uint8_t *)validation); + if (!check_dst_port(sport, num_ports, validation)) { + return PACKET_INVALID; + } + } else { + return PACKET_INVALID; + } + return PACKET_VALID; +} + +static void synackscan_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = get_ip_header(packet, len); + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len); + fs_add_uint64(fs, "sport", (uint64_t)ntohs(tcp->th_sport)); + fs_add_uint64(fs, "dport", (uint64_t)ntohs(tcp->th_dport)); + fs_add_uint64(fs, "seqnum", (uint64_t)ntohl(tcp->th_seq)); + fs_add_uint64(fs, "acknum", (uint64_t)ntohl(tcp->th_ack)); + fs_add_uint64(fs, "window", (uint64_t)ntohs(tcp->th_win)); + if (tcp->th_flags & TH_RST) { // RST packet + fs_add_constchar(fs, "classification", "rst"); + } else { // SYNACK packet + fs_add_constchar(fs, "classification", "synack"); + } + fs_add_bool(fs, "success", 1); + fs_add_null_icmp(fs); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + // tcp + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "seqnum"); + fs_add_null(fs, "acknum"); + fs_add_null(fs, "window"); + // global + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + // icmp + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window"}, + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + ICMP_FIELDSET_FIELDS, +}; + +probe_module_t module_tcp_synackscan = { + .name = "tcp_synackscan", + .max_packet_length = ZMAP_TCP_SYNACKSCAN_PACKET_LEN, + .pcap_filter = "(tcp && tcp[13] & 4 != 0 || tcp[13] == 18) || icmp", + .pcap_snaplen = 96, + .port_args = 1, + .global_initialize = &synackscan_global_initialize, + .thread_initialize = &synackscan_init_perthread, + .make_packet = &synackscan_make_packet, + .print_packet = &synscan_print_packet, + .process_packet = &synackscan_process_packet, + .validate_packet = &synackscan_validate_packet, + .close = NULL, + .helptext = "Probe module that sends a TCP SYNACK packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a failure and a reset packet " + "is considered a success.", + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0])}; diff --git a/src/probe_modules/module_tcp_synscan.c b/src/probe_modules/module_tcp_synscan.c new file mode 100644 index 0000000..09b4204 --- /dev/null +++ b/src/probe_modules/module_tcp_synscan.c @@ -0,0 +1,234 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" +#include "validate.h" + +#define ZMAP_TCP_SYNSCAN_TCP_HEADER_LEN 24 +#define ZMAP_TCP_SYNSCAN_PACKET_LEN 58 + +probe_module_t module_tcp_synscan; + +static uint16_t num_source_ports; + +static int synscan_global_initialize(struct state_conf *state) +{ + num_source_ports = + state->source_port_last - state->source_port_first + 1; + return EXIT_SUCCESS; +} + +static int synscan_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = + htons(sizeof(struct ip) + ZMAP_TCP_SYNSCAN_TCP_HEADER_LEN); + make_ip_header(ip_header, IPPROTO_TCP, len); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + make_tcp_header(tcp_header, TH_SYN); + set_mss_option(tcp_header); + return EXIT_SUCCESS; +} + +static int synscan_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, port_n_t dport, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + uint32_t tcp_seq = validation[0]; + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + port_h_t sport = get_src_port(num_source_ports, probe_num, validation); + tcp_header->th_sport = htons(sport); + tcp_header->th_dport = dport; + tcp_header->th_seq = tcp_seq; + // checksum value must be zero when calculating packet's checksum + tcp_header->th_sum = 0; + tcp_header->th_sum = tcp_checksum(ZMAP_TCP_SYNSCAN_TCP_HEADER_LEN, + ip_header->ip_src.s_addr, + ip_header->ip_dst.s_addr, tcp_header); + // checksum value must be zero when calculating packet's checksum + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + *buf_len = ZMAP_TCP_SYNSCAN_PACKET_LEN; + return EXIT_SUCCESS; +} + +// not static because used by synack scan +void synscan_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct tcphdr *tcph = (struct tcphdr *)&iph[1]; + fprintf(fp, + "tcp { source: %u | dest: %u | seq: %u | checksum: %#04X }\n", + ntohs(tcph->th_sport), ntohs(tcph->th_dport), + ntohl(tcph->th_seq), ntohs(tcph->th_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +static int synscan_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation, + const struct port_conf *ports) +{ + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len); + if (!tcp) { + return PACKET_INVALID; + } + port_h_t sport = ntohs(tcp->th_sport); + port_h_t dport = ntohs(tcp->th_dport); + // validate source port + if (!check_src_port(sport, ports)) { + return PACKET_INVALID; + } + // validate destination port + if (!check_dst_port(dport, num_source_ports, validation)) { + return PACKET_INVALID; + } + // check whether we'll ever send to this IP during the scan + if (!blocklist_is_allowed(*src_ip)) { + return PACKET_INVALID; + } + // We treat RST packets different from non RST packets + if (tcp->th_flags & TH_RST) { + // For RST packets, recv(ack) == sent(seq) + 0 or + 1 + if (htonl(tcp->th_ack) != htonl(validation[0]) && + htonl(tcp->th_ack) != htonl(validation[0]) + 1) { + return PACKET_INVALID; + } + } else { + // For non RST packets, recv(ack) == sent(seq) + 1 + if (htonl(tcp->th_ack) != htonl(validation[0]) + 1) { + return PACKET_INVALID; + } + } + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + struct ip *ip_inner; + size_t ip_inner_len; + if (icmp_helper_validate(ip_hdr, len, sizeof(struct tcphdr), + &ip_inner, + &ip_inner_len) == PACKET_INVALID) { + return PACKET_INVALID; + } + struct tcphdr *tcp = get_tcp_header(ip_inner, ip_inner_len); + if (!tcp) { + return PACKET_INVALID; + } + // we can always check the destination port because this is the + // original packet and wouldn't have been altered by something + // responding on a different port. Note this is *different* + // than the logic above because we're validating the probe packet + // rather than the response packet + port_h_t sport = ntohs(tcp->th_sport); + port_h_t dport = ntohs(tcp->th_dport); + if (!check_src_port(dport, ports)) { + return PACKET_INVALID; + } + validate_gen(ip_hdr->ip_dst.s_addr, ip_inner->ip_dst.s_addr, + tcp->th_dport, (uint8_t *)validation); + if (!check_dst_port(sport, num_source_ports, validation)) { + return PACKET_INVALID; + } + } else { + return PACKET_INVALID; + } + return PACKET_VALID; +} + +static void synscan_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = get_ip_header(packet, len); + assert(ip_hdr); + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len); + assert(tcp); + fs_add_uint64(fs, "sport", (uint64_t)ntohs(tcp->th_sport)); + fs_add_uint64(fs, "dport", (uint64_t)ntohs(tcp->th_dport)); + fs_add_uint64(fs, "seqnum", (uint64_t)ntohl(tcp->th_seq)); + fs_add_uint64(fs, "acknum", (uint64_t)ntohl(tcp->th_ack)); + fs_add_uint64(fs, "window", (uint64_t)ntohs(tcp->th_win)); + if (tcp->th_flags & TH_RST) { // RST packet + fs_add_constchar(fs, "classification", "rst"); + fs_add_bool(fs, "success", 0); + } else { // SYNACK packet + fs_add_constchar(fs, "classification", "synack"); + fs_add_bool(fs, "success", 1); + } + fs_add_null_icmp(fs); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + // tcp + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "seqnum"); + fs_add_null(fs, "acknum"); + fs_add_null(fs, "window"); + // global + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + // icmp + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window"}, + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + ICMP_FIELDSET_FIELDS, +}; + +probe_module_t module_tcp_synscan = { + .name = "tcp_synscan", + .max_packet_length = ZMAP_TCP_SYNSCAN_PACKET_LEN, + .pcap_filter = "(tcp && tcp[13] & 4 != 0 || tcp[13] == 18) || icmp", + .pcap_snaplen = 96, + .port_args = 1, + .global_initialize = &synscan_global_initialize, + .thread_initialize = &synscan_init_perthread, + .make_packet = &synscan_make_packet, + .print_packet = &synscan_print_packet, + .process_packet = &synscan_process_packet, + .validate_packet = &synscan_validate_packet, + .close = NULL, + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response.", + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0])}; diff --git a/src/probe_modules/module_tcp_synscan.h b/src/probe_modules/module_tcp_synscan.h new file mode 100644 index 0000000..b23af83 --- /dev/null +++ b/src/probe_modules/module_tcp_synscan.h @@ -0,0 +1,38 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" + +int synscan_global_initialize(struct state_conf *state); + +int synscan_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + port_h_t dst_port, UNUSED void **arg_ptr); + +int synscan_make_packet(void *buf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip, + uint8_t ttl, uint32_t *validation, int probe_num, + UNUSED void *arg); + +void synscan_print_packet(FILE *fp, void *packet); + +int synscan_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED uint32_t *src_ip, uint32_t *validation); + +void synscan_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, UNUSED uint32_t *validation); diff --git a/src/probe_modules/module_udp.c b/src/probe_modules/module_udp.c new file mode 100644 index 0000000..8cf7f29 --- /dev/null +++ b/src/probe_modules/module_udp.c @@ -0,0 +1,869 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* send module for performing arbitrary UDP scans */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "../../lib/blocklist.h" +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" +#include "../../lib/lockfd.h" +#include "logger.h" +#include "probe_modules.h" +#include "packet.h" +#include "aesrand.h" +#include "state.h" +#include "module_udp.h" + +#define MAX_UDP_PAYLOAD_LEN 1472 +#define ICMP_HEADER_SIZE 8 + +static uint8_t *udp_fixed_payload = NULL; +static size_t udp_fixed_payload_len = 0; + +static udp_payload_template_t *udp_template = NULL; + +const char *udp_usage_error = + "unknown UDP probe specification (expected file:/path or text:STRING or hex:01020304 or template:/path or template-fields)"; + +const unsigned char *charset_alphanum = + (unsigned char + *)"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +const unsigned char *charset_alpha = + (unsigned char *)"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; +const unsigned char *charset_digit = (unsigned char *)"0123456789"; +const unsigned char charset_all[257] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, + 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, + 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, + 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, + 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, + 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, + 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, + 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, + 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, + 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, + 0xfd, 0xfe, 0xff, 0x00}; + +static int num_ports; + +probe_module_t module_udp; + +// Field definitions for template parsing and displaying usage +static uint32_t udp_num_template_field_types = 12; +static udp_payload_field_type_def_t udp_payload_template_fields[] = { + {.name = "SADDR_N", + .ftype = UDP_SADDR_N, + .max_length = 4, + .desc = "Source IP address in network byte order"}, + {.name = "SADDR", + .ftype = UDP_SADDR_A, + .max_length = 15, + .desc = "Source IP address in dotted-quad format"}, + {.name = "DADDR_N", + .ftype = UDP_DADDR_N, + .max_length = 4, + .desc = "Destination IP address in network byte order"}, + {.name = "DADDR", + .ftype = UDP_DADDR_A, + .max_length = 15, + .desc = "Destination IP address in dotted-quad format"}, + {.name = "SPORT_N", + .ftype = UDP_SPORT_N, + .max_length = 2, + .desc = "UDP source port in network byte order"}, + {.name = "SPORT", + .ftype = UDP_SPORT_A, + .max_length = 5, + .desc = "UDP source port in ascii format"}, + {.name = "DPORT_N", + .ftype = UDP_DPORT_N, + .max_length = 2, + .desc = "UDP destination port in network byte order"}, + {.name = "DPORT", + .ftype = UDP_DPORT_A, + .max_length = 5, + .desc = "UDP destination port in ascii format"}, + {.name = "RAND_BYTE", + .ftype = UDP_RAND_BYTE, + .max_length = 0, + .desc = "Random bytes from 0-255"}, + {.name = "RAND_DIGIT", + .ftype = UDP_RAND_DIGIT, + .max_length = 0, + .desc = "Random digits from 0-9"}, + {.name = "RAND_ALPHA", + .ftype = UDP_RAND_ALPHA, + .max_length = 0, + .desc = "Random mixed-case letters (a-z)"}, + {.name = "RAND_ALPHANUM", + .ftype = UDP_RAND_ALPHANUM, + .max_length = 0, + .desc = "Random mixed-case letters (a-z) and numbers"}}; + +void udp_set_num_ports(int x) { num_ports = x; } + +int udp_global_initialize(struct state_conf *conf) +{ + uint32_t udp_template_max_len = 0; + num_ports = conf->source_port_last - conf->source_port_first + 1; + + if (!conf->probe_args) { + log_error( + "udp", "%s", + "--probe-args are required, run --probe-module=udp --help for a longer description of the arguments"); + return EXIT_FAILURE; + } + + const char *args = conf->probe_args; + if (strcmp(args, "template-fields") == 0) { + lock_file(stderr); + fprintf( + stderr, "%s", + "List of allowed UDP template fields (name: description)\n\n"); + for (uint32_t i = 0; i < udp_num_template_field_types; ++i) { + fprintf(stderr, "%s: %s\n", + udp_payload_template_fields[i].name, + udp_payload_template_fields[i].desc); + } + fprintf(stderr, "%s\n", ""); + fflush(stderr); + unlock_file(stderr); + exit(0); + } + + const char *c = strchr(args, ':'); + if (!c) { + log_fatal("udp", udp_usage_error); + } + size_t arg_name_len = c - args; + c++; + if (strncmp(args, "text", arg_name_len) == 0) { + udp_fixed_payload = (uint8_t *)strdup(c); + udp_fixed_payload_len = strlen(c); + } else if (strncmp(args, "file", arg_name_len) == 0) { + udp_fixed_payload = xmalloc(MAX_UDP_PAYLOAD_LEN); + FILE *f = fopen(c, "rb"); + if (!f) { + log_fatal("udp", "could not open UDP data file '%s'\n", + c); + } + udp_fixed_payload_len = + fread(udp_fixed_payload, 1, MAX_UDP_PAYLOAD_LEN, f); + fclose(f); + } else if (strncmp(args, "template", arg_name_len) == 0) { + uint8_t in[MAX_UDP_PAYLOAD_LEN]; + FILE *f = fopen(c, "rb"); + if (!f) { + log_fatal("udp", "could not open UDP data file '%s'\n", + c); + } + size_t in_len = fread(in, 1, MAX_UDP_PAYLOAD_LEN, f); + fclose(f); + + udp_template = + udp_template_load(in, in_len, &udp_template_max_len); + module_udp.make_packet = udp_make_templated_packet; + } else if (strncmp(args, "hex", arg_name_len) == 0) { + udp_fixed_payload_len = strlen(c) / 2; + udp_fixed_payload = xmalloc(udp_fixed_payload_len); + + unsigned int n; + for (size_t i = 0; i < udp_fixed_payload_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + log_fatal("udp", "non-hex character: '%c'", + c[i * 2]); + } + udp_fixed_payload[i] = (n & 0xff); + } + } else { + log_fatal("udp", udp_usage_error); + } + + if (udp_fixed_payload_len > MAX_UDP_PAYLOAD_LEN) { + log_warn("udp", + "warning: reducing fixed UDP payload to %d " + "bytes (from %d) to fit on the wire\n", + MAX_UDP_PAYLOAD_LEN, udp_fixed_payload_len); + udp_fixed_payload_len = MAX_UDP_PAYLOAD_LEN; + } + + size_t header_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr); + if (udp_fixed_payload_len > 0) { + module_udp.max_packet_length = + header_len + udp_fixed_payload_len; + } else if (udp_template_max_len > 0) { + module_udp.max_packet_length = + header_len + udp_template_max_len; + } + assert(module_udp.max_packet_length); + assert(module_udp.max_packet_length <= MAX_PACKET_SIZE); + return EXIT_SUCCESS; +} + +int udp_global_cleanup(UNUSED struct state_conf *zconf, + UNUSED struct state_send *zsend, + UNUSED struct state_recv *zrecv) +{ + if (udp_fixed_payload) { + free(udp_fixed_payload); + udp_fixed_payload = NULL; + } + if (udp_template) { + udp_template_free(udp_template); + udp_template = NULL; + } + return EXIT_SUCCESS; +} + +int udp_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + + udp_fixed_payload_len); + make_ip_header(ip_header, IPPROTO_UDP, ip_len); + + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + uint16_t udp_len = sizeof(struct udphdr) + udp_fixed_payload_len; + make_udp_header(udp_header, udp_len); + + if (udp_fixed_payload) { + void *payload = &udp_header[1]; + memcpy(payload, udp_fixed_payload, udp_fixed_payload_len); + } + + // Seed our random number generator with the global generator + uint32_t seed = aesrand_getword(zconf.aes); + aesrand_t *aes = aesrand_init_from_seed(seed); + *arg_ptr = aes; + + return EXIT_SUCCESS; +} + +int udp_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, port_n_t dport, uint8_t ttl, + uint32_t *validation, int probe_num, UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)&ip_header[1]; + size_t headers_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr); + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + udp_header->uh_sport = + htons(get_src_port(num_ports, probe_num, validation)); + udp_header->uh_dport = dport; + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + // Output the total length of the packet + *buf_len = headers_len + udp_fixed_payload_len; + return EXIT_SUCCESS; +} + +int udp_make_templated_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, port_n_t dport, uint8_t ttl, + uint32_t *validation, int probe_num, void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)&ip_header[1]; + size_t headers_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr); + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + udp_header->uh_sport = + htons(get_src_port(num_ports, probe_num, validation)); + udp_header->uh_dport = dport; + + char *payload = (char *)&udp_header[1]; + memset(payload, 0, MAX_UDP_PAYLOAD_LEN); + + // Grab our random number generator + aesrand_t *aes = (aesrand_t *)arg; + + // The buf is a stack var of our caller of size MAX_PACKET_SIZE + // Recalculate the payload using the loaded template + int payload_len = + udp_template_build(udp_template, payload, MAX_UDP_PAYLOAD_LEN, + ip_header, udp_header, aes); + + // If success is zero, the template output was truncated + if (payload_len <= 0) { + log_fatal("udp", + "UDP payload template generated an empty payload"); + } + + // Update the IP and UDP headers to match the new payload length + ip_header->ip_len = + htons(sizeof(struct ip) + sizeof(struct udphdr) + payload_len); + udp_header->uh_ulen = ntohs(sizeof(struct udphdr) + payload_len); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + // Recalculate the total length of the packet + *buf_len = headers_len + payload_len; + return EXIT_SUCCESS; +} + +void udp_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct udphdr *udph = (struct udphdr *)(&iph[1]); + fprintf(fp, "udp { source: %u | dest: %u | checksum: %#04X }\n", + ntohs(udph->uh_sport), ntohs(udph->uh_dport), + ntohs(udph->uh_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +void udp_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + fs_add_constchar(fs, "classification", "udp"); + fs_add_bool(fs, "success", 1); + fs_add_uint64(fs, "sport", ntohs(udp->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp->uh_dport)); + fs_add_uint64(fs, "udp_pkt_size", ntohs(udp->uh_ulen)); + // Verify that the UDP length is big enough for the header and + // at least one byte + uint16_t data_len = ntohs(udp->uh_ulen); + if (data_len > sizeof(struct udphdr)) { + uint32_t overhead = + (sizeof(struct udphdr) + (ip_hdr->ip_hl * 4)); + uint32_t max_rlen = len - overhead; + uint32_t max_ilen = ntohs(ip_hdr->ip_len) - overhead; + + // Verify that the UDP length is inside of our received + // buffer + if (data_len > max_rlen) { + data_len = max_rlen; + } + // Verify that the UDP length is inside of our IP packet + if (data_len > max_ilen) { + data_len = max_ilen; + } + fs_add_binary(fs, "data", data_len, (void *)&udp[1], 0); + // Some devices reply with a zero UDP length but still + // return data, ignore the data + } else { + fs_add_null(fs, "data"); + } + fs_add_null_icmp(fs); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "udp_pkt_size"); + fs_add_null(fs, "data"); + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + } else { + fs_add_constchar(fs, "classification", "other"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "udp_pkt_size"); + fs_add_null(fs, "data"); + fs_add_null_icmp(fs); + } +} + +int udp_validate_packet(const struct ip *ip_hdr, uint32_t len, uint32_t *src_ip, + uint32_t *validation, const struct port_conf *ports) +{ + return udp_do_validate_packet(ip_hdr, len, src_ip, validation, + num_ports, NO_SRC_PORT_VALIDATION, ports); +} + +// Do very basic validation that this is an ICMP response to a packet we sent +// Find the application layer packet that was originally sent and give it back +// to the caller to do additional validation (e.g., correct TCP destination +// port) + +int udp_do_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation, + int num_ports, int validate_port, + const struct port_conf *ports) +{ + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + if (!udp) { + return PACKET_INVALID; + } + uint16_t dport = ntohs(udp->uh_dport); + if (!check_dst_port(dport, num_ports, validation)) { + return PACKET_INVALID; + } + if (!blocklist_is_allowed(*src_ip)) { + return PACKET_INVALID; + } + if (validate_port == SRC_PORT_VALIDATION) { + uint16_t sport = ntohs(udp->uh_sport); + if (!check_src_port(sport, ports)) { + return PACKET_INVALID; + } + } + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + struct ip *ip_inner; + size_t ip_inner_len; + if (icmp_helper_validate(ip_hdr, len, sizeof(struct udphdr), + &ip_inner, + &ip_inner_len) == PACKET_INVALID) { + return PACKET_INVALID; + } + struct udphdr *udp = get_udp_header(ip_inner, ip_inner_len); + // we can always check the destination port because this is the + // original packet and wouldn't have been altered by something + // responding on a different port + uint16_t dport = ntohs(udp->uh_dport); + uint16_t sport = ntohs(udp->uh_sport); + if (!check_src_port(dport, ports)) { + return PACKET_INVALID; + } + if (!check_dst_port(sport, num_ports, validation)) { + return PACKET_INVALID; + } + } else { + return PACKET_INVALID; + } + return PACKET_VALID; +} + +// Add a new field to the template +void udp_template_add_field(udp_payload_template_t *t, + udp_payload_field_type_t ftype, unsigned int length, + char *data) +{ + udp_payload_field_t *c; + + t->fcount++; + t->fields = + xrealloc(t->fields, sizeof(udp_payload_field_t) * t->fcount); + t->fields[t->fcount - 1] = xmalloc(sizeof(udp_payload_field_t)); + c = t->fields[t->fcount - 1]; + assert(c); + c->ftype = ftype; + c->length = length; + c->data = data; +} + +// Free all buffers held by the payload template, including its own +void udp_template_free(udp_payload_template_t *t) +{ + for (unsigned int x = 0; x < t->fcount; x++) { + if (t->fields[x]->data) { + free(t->fields[x]->data); + t->fields[x]->data = NULL; + } + free(t->fields[x]); + t->fields[x] = NULL; + } + free(t->fields); + t->fields = NULL; + t->fcount = 0; + free(t); +} + +int udp_random_bytes(char *dst, int len, const unsigned char *charset, + int charset_len, aesrand_t *aes) +{ + int i; + for (i = 0; i < len; i++) { + *dst++ = + charset[(aesrand_getword(aes) & 0xFFFFFFFF) % charset_len]; + } + return i; +} + +int udp_template_build(udp_payload_template_t *t, char *out, unsigned int len, + struct ip *ip_hdr, struct udphdr *udp_hdr, + aesrand_t *aes) +{ + udp_payload_field_t *c; + char *p; + char *max; + char tmp[256]; + int full = 0; + unsigned int x, y; + uint32_t *u32; + uint16_t *u16; + + max = out + len; + p = out; + + for (x = 0; x < t->fcount; x++) { + c = t->fields[x]; + + // Exit the processing loop if our packet buffer would overflow + if (p + c->length >= max) { + full = 1; + return 0; + } + + switch (c->ftype) { + // These fields have a specified output length value + + case UDP_DATA: + if (!(c->data && c->length)) + break; + memcpy(p, c->data, c->length); + p += c->length; + break; + + case UDP_RAND_DIGIT: + p += udp_random_bytes(p, c->length, charset_digit, 10, + aes); + break; + + case UDP_RAND_ALPHA: + p += udp_random_bytes(p, c->length, charset_alpha, 52, + aes); + break; + + case UDP_RAND_ALPHANUM: + p += udp_random_bytes(p, c->length, charset_alphanum, + 62, aes); + break; + + case UDP_RAND_BYTE: + p += udp_random_bytes(p, c->length, charset_all, 256, + aes); + break; + + // These fields need to calculate size on their own + + // TODO: Condense these case statements to remove redundant code + case UDP_SADDR_A: + if (p + 15 >= max) { + full = 1; + break; + } + // Write to stack and then memcpy in order to properly + // track length + inet_ntop(AF_INET, (char *)&ip_hdr->ip_src, tmp, + sizeof(tmp) - 1); + memcpy(p, tmp, strlen(tmp)); + p += strlen(tmp); + break; + + case UDP_DADDR_A: + if (p + 15 >= max) { + full = 1; + break; + } + // Write to stack and then memcpy in order to properly + // track length + inet_ntop(AF_INET, (char *)&ip_hdr->ip_dst, tmp, + sizeof(tmp) - 1); + memcpy(p, tmp, strlen(tmp)); + p += strlen(tmp); + break; + + case UDP_SADDR_N: + if (p + 4 >= max) { + full = 1; + break; + } + + u32 = (uint32_t *)p; + *u32 = ip_hdr->ip_src.s_addr; + p += 4; + break; + + case UDP_DADDR_N: + if (p + 4 >= max) { + full = 1; + break; + } + u32 = (uint32_t *)p; + *u32 = ip_hdr->ip_dst.s_addr; + p += 4; + break; + + case UDP_SPORT_N: + if (p + 2 >= max) { + full = 1; + break; + } + u16 = (uint16_t *)p; + *u16 = udp_hdr->uh_sport; + p += 2; + break; + + case UDP_DPORT_N: + if (p + 2 >= max) { + full = 1; + break; + } + u16 = (uint16_t *)p; + *u16 = udp_hdr->uh_dport; + p += 2; + break; + + case UDP_SPORT_A: + if (p + 5 >= max) { + full = 1; + break; + } + y = snprintf(tmp, 6, "%d", ntohs(udp_hdr->uh_sport)); + memcpy(p, tmp, y); + p += y; + break; + + case UDP_DPORT_A: + if (p + 5 >= max) { + full = 1; + break; + } + y = snprintf(tmp, 6, "%d", ntohs(udp_hdr->uh_dport)); + memcpy(p, tmp, y); + p += y; + break; + } + + // Bail out if our packet buffer would overflow + if (full == 1) { + return 0; + } + } + + return p - out; +} + +// Convert a string field name to a field type, parsing any specified length +// value +int udp_template_field_lookup(const char *vname, udp_payload_field_t *c) +{ + static const size_t fcount = sizeof(udp_payload_template_fields) / + sizeof(udp_payload_template_fields[0]); + size_t vname_len = strlen(vname); + size_t type_name_len = vname_len; + const char *param = strstr(vname, "="); + if (param) { + type_name_len = param - vname; + param++; + } + + // Most field types treat their parameter as a generator output length + // unless it is ignored (ADDR, PORT, etc). + long olen = 0; + if (param && !*param) { + log_fatal( + "udp", + "invalid template: field spec %s is invalid (missing length)", + vname); + } + if (param) { + char *end = NULL; + errno = 0; + olen = strtol(param, &end, 10); + if (errno) { + log_fatal( + "udp", + "invalid template: unable to read length from %s: %s", + vname, strerror(errno)); + } + if (!end || end != vname + vname_len) { + log_fatal( + "udp", + "invalid template: unable to read length from %s", + vname); + } + if (olen < 0 || olen > MAX_UDP_PAYLOAD_LEN) { + log_fatal( + "udp", + "invalid template: field size %d is larger than the max (%d)", + olen, MAX_UDP_PAYLOAD_LEN); + } + } + + // Find a field that matches the + for (unsigned int f = 0; f < fcount; f++) { + const udp_payload_field_type_def_t *ftype = + &udp_payload_template_fields[f]; + if (strncmp(vname, ftype->name, type_name_len) == 0 && + strlen(ftype->name) == type_name_len) { + c->ftype = ftype->ftype; + c->length = ftype->max_length ? ftype->max_length + : (size_t)olen; + c->data = NULL; + return 1; + } + } + + // No match, skip and treat it as a data field + return 0; +} + +// Allocate a payload template and populate it by parsing a template file as a +// binary buffer +udp_payload_template_t *udp_template_load(uint8_t *buf, uint32_t buf_len, + uint32_t *max_pkt_len) +{ + udp_payload_template_t *t = xmalloc(sizeof(udp_payload_template_t)); + uint32_t _max_pkt_len = 0; + + // The last $ we encountered outside of a field specifier + uint8_t *dollar = NULL; + + // The last { we encountered outside of a field specifier + uint8_t *lbrack = NULL; + + // Track the start pointer of a data field (static) + uint8_t *s = buf; + + // Track the index into the template + uint8_t *p = buf; + + char *tmp; + unsigned int tlen; + + udp_payload_field_t c; + + t->fcount = 0; + t->fields = NULL; + + while (p < (buf + buf_len)) { + switch (*p) { + case '$': + if ((dollar && !lbrack) || !dollar) { + dollar = p; + } + p++; + continue; + case '{': + if (dollar && !lbrack) { + lbrack = p; + } + + p++; + continue; + case '}': + if (!(dollar && lbrack)) { + p++; + continue; + } + + // Store the leading bytes before ${ as a data field + tlen = dollar - s; + if (tlen > 0) { + tmp = xmalloc(tlen); + memcpy(tmp, s, tlen); + udp_template_add_field(t, UDP_DATA, tlen, tmp); + _max_pkt_len += tlen; + } + + tmp = xcalloc(1, p - lbrack); + memcpy(tmp, lbrack + 1, p - lbrack - 1); + + if (udp_template_field_lookup(tmp, &c)) { + udp_template_add_field(t, c.ftype, c.length, + c.data); + _max_pkt_len += c.length; + // Push the pointer past the } if this was a + // valid variable + s = p + 1; + } else { + + // Rewind back to the ${ sequence if this was an + // invalid variable + s = dollar; + } + + free(tmp); + break; + default: + if (dollar && lbrack) { + p++; + continue; + } + } + dollar = NULL; + lbrack = NULL; + p++; + } + + // Store the trailing bytes as a final data field + if (s < p) { + tlen = p - s; + tmp = xmalloc(tlen); + memcpy(tmp, s, tlen); + udp_template_add_field(t, UDP_DATA, tlen, tmp); + _max_pkt_len += tlen; + } + *max_pkt_len = _max_pkt_len; + return t; +} + +static fielddef_t fields[] = { + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + {.name = "udp_pkt_size", .type = "int", .desc = "UDP packet length"}, + {.name = "data", .type = "binary", .desc = "UDP payload"}, + ICMP_FIELDSET_FIELDS, +}; + +probe_module_t module_udp = { + .name = "udp", + .max_packet_length = 0, // set in init + .pcap_filter = "udp || icmp", + .pcap_snaplen = + MAX_UDP_PAYLOAD_LEN + 20 + 24, // Ether Header, IP Header with Options + .port_args = 1, + .thread_initialize = &udp_init_perthread, + .global_initialize = &udp_global_initialize, + .make_packet = + &udp_make_packet, // can be overridden to udp_make_templated_packet by udp_global_initalize + .print_packet = &udp_print_packet, + .validate_packet = &udp_validate_packet, + .process_packet = &udp_process_packet, + .close = &udp_global_cleanup, + .helptext = "Probe module that sends UDP packets to hosts. Packets can " + "optionally be templated based on destination host. Specify " + "packet file with --probe-args=file:/path_to_packet_file " + "and templates with template:/path_to_template_file.", + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0])}; diff --git a/src/probe_modules/module_udp.h b/src/probe_modules/module_udp.h new file mode 100644 index 0000000..c94a4bb --- /dev/null +++ b/src/probe_modules/module_udp.h @@ -0,0 +1,96 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include + +#include + +#include "aesrand.h" +#include "types.h" + +#include "state.h" + +#define NO_SRC_PORT_VALIDATION 0 +#define SRC_PORT_VALIDATION 1 + +typedef enum udp_payload_field_type { + UDP_DATA, + UDP_SADDR_N, + UDP_SADDR_A, + UDP_DADDR_N, + UDP_DADDR_A, + UDP_SPORT_N, + UDP_SPORT_A, + UDP_DPORT_N, + UDP_DPORT_A, + UDP_RAND_BYTE, + UDP_RAND_DIGIT, + UDP_RAND_ALPHA, + UDP_RAND_ALPHANUM +} udp_payload_field_type_t; + +typedef struct udp_payload_field_type_def { + const char *name; + const char *desc; + size_t max_length; + udp_payload_field_type_t ftype; +} udp_payload_field_type_def_t; + +typedef struct udp_payload_field { + enum udp_payload_field_type ftype; + size_t length; + char *data; +} udp_payload_field_t; + +typedef struct udp_payload_template { + unsigned int fcount; + struct udp_payload_field **fields; +} udp_payload_template_t; + +typedef struct udp_payload_output { + int length; + char *data; +} udp_payload_output_t; + +void udp_print_packet(FILE *fp, void *packet); + +int udp_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, port_n_t dport, uint8_t ttl, + uint32_t *validation, int probe_num, void *arg); +int udp_make_templated_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, port_n_t dport, uint8_t ttl, + uint32_t *validation, int probe_num, void *arg); + +int udp_do_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED uint32_t *src_ip, uint32_t *validation, + int num_ports, int expected_port, + const struct port_conf *ports); + +void udp_set_num_ports(int); +int udp_global_initialize(struct state_conf *conf); +int udp_global_cleanup(UNUSED struct state_conf *zconf, + UNUSED struct state_send *zsend, + UNUSED struct state_recv *zrecv); + +void udp_template_add_field(udp_payload_template_t *t, + udp_payload_field_type_t ftype, unsigned int length, + char *data); + +void udp_template_free(udp_payload_template_t *t); + +int udp_template_build(udp_payload_template_t *t, char *out, unsigned int len, + struct ip *ip_hdr, struct udphdr *udp_hdr, + aesrand_t *aes); + +int udp_template_field_lookup(const char *vname, udp_payload_field_t *c); + +udp_payload_template_t *udp_template_load(uint8_t *buf, uint32_t buf_len, + uint32_t *max_pkt_len); diff --git a/src/probe_modules/module_upnp.c b/src/probe_modules/module_upnp.c new file mode 100644 index 0000000..956d3d6 --- /dev/null +++ b/src/probe_modules/module_upnp.c @@ -0,0 +1,272 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../../lib/logger.h" +#include "../../lib/xalloc.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" +#include "module_udp.h" + +#define ICMP_UNREACH_HEADER_SIZE 8 + +static const char *upnp_query = "M-SEARCH * HTTP/1.1\r\n" + "Host:239.255.255.250:1900\r\n" + "ST:upnp:rootdevice\r\n" + "Man:\"ssdp:discover\"\r\nMX:3\r\n\r\n"; + +probe_module_t module_upnp; + +static int num_ports; + +int upnp_global_initialize(struct state_conf *state) +{ + num_ports = state->source_port_last - state->source_port_first + 1; + udp_set_num_ports(num_ports); + return EXIT_SUCCESS; +} + +int upnp_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + + uint16_t len = htons(sizeof(struct ip) + sizeof(struct udphdr) + + strlen(upnp_query)); + make_ip_header(ip_header, IPPROTO_UDP, len); + + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + len = sizeof(struct udphdr) + strlen(upnp_query); + make_udp_header(udp_header, len); + + char *payload = (char *)(&udp_header[1]); + + assert(sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr) + strlen(upnp_query) <= + MAX_PACKET_SIZE); + + assert(MAX_PACKET_SIZE - ((char *)payload - (char *)buf) > + (int)strlen(upnp_query)); + strcpy(payload, upnp_query); + + return EXIT_SUCCESS; +} + +int upnp_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation, + const struct port_conf *ports) +{ + return udp_do_validate_packet(ip_hdr, len, src_ip, validation, + num_ports, SRC_PORT_VALIDATION, ports); +} + +void upnp_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = + (struct udphdr *)((char *)ip_hdr + ip_hdr->ip_hl * 4); + + char *payload = (char *)(&udp[1]); + uint16_t plen = udp->uh_ulen - 8; + + char *s = xmalloc(plen + 1); + strncpy(s, payload, plen); + s[plen] = 0; + + int is_first = 1; + const char *classification = "none"; + uint64_t is_success = 0; + + char *server = NULL, *location = NULL, *usn = NULL, *st = NULL, + *cachecontrol = NULL, *ext = NULL, *xusragent = NULL, + *date = NULL, *agent = NULL; + + char *pch = strtok(s, "\n"); + while (pch != NULL) { + if (pch[strlen(pch) - 1] == '\r') { + pch[strlen(pch) - 1] = '\0'; + } + if (strlen(pch) == 0) { + pch = strtok(NULL, "\n"); + continue; + } + // the first pch is always supposed to be an HTTP + // response + if (is_first) { + if (strcmp(pch, "HTTP/1.1 200 OK")) { + classification = "no-http-header"; + is_success = 0; + goto cleanup; + } + is_first = 0; + is_success = 1; + classification = "upnp"; + pch = strtok(NULL, "\n"); + continue; + } + char *value = pch; + char *key = strsep(&value, ":"); + if (!key) { + pch = strtok(NULL, "\n"); + continue; + } + if (!value) { + pch = strtok(NULL, "\n"); + continue; + } + if (value[0] == ' ') { + value += (size_t)1; + } + if (!strcasecmp(key, "server")) { + server = strdup(value); + } else if (!strcasecmp(key, "location")) { + location = strdup(value); + } else if (!strcasecmp(key, "USN")) { + usn = strdup(value); + } else if (!strcasecmp(key, "EXT")) { + ext = strdup(value); + } else if (!strcasecmp(key, "ST")) { + st = strdup(value); + } else if (!strcasecmp(key, "Agent")) { + agent = strdup(value); + } else if (!strcasecmp(key, "X-User-Agent")) { + xusragent = strdup(value); + } else if (!strcasecmp(key, "date")) { + date = strdup(value); + } else if (!strcasecmp(key, "Cache-Control")) { + cachecontrol = strdup(value); + } else { + // log_debug("upnp-module", "new key: %s", key); + } + pch = strtok(NULL, "\n"); + } + + cleanup: + fs_add_string(fs, "classification", (char *)classification, 0); + fs_add_bool(fs, "success", is_success); + fs_chkadd_unsafe_string(fs, "server", server, 1); + fs_chkadd_unsafe_string(fs, "location", location, 1); + fs_chkadd_unsafe_string(fs, "usn", usn, 1); + fs_chkadd_unsafe_string(fs, "st", st, 1); + fs_chkadd_unsafe_string(fs, "ext", ext, 1); + fs_chkadd_unsafe_string(fs, "cache_control", cachecontrol, 1); + fs_chkadd_unsafe_string(fs, "x_user_agent", xusragent, 1); + fs_chkadd_unsafe_string(fs, "agent", agent, 1); + fs_chkadd_unsafe_string(fs, "date", date, 1); + fs_add_uint64(fs, "sport", ntohs(udp->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp->uh_dport)); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); + + fs_add_binary(fs, "data", + (ntohs(udp->uh_ulen) - sizeof(struct udphdr)), + (void *)&udp[1], 0); + + free(s); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + fs_add_constchar(fs, "classification", "icmp"); + fs_add_uint64(fs, "success", 0); + + fs_add_null(fs, "server"); + fs_add_null(fs, "location"); + fs_add_null(fs, "usn"); + fs_add_null(fs, "st"); + fs_add_null(fs, "ext"); + fs_add_null(fs, "cache_control"); + fs_add_null(fs, "x_user_agent"); + fs_add_null(fs, "agent"); + fs_add_null(fs, "date"); + + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + fs_add_null(fs, "data"); + } else { + fs_add_constchar(fs, "classification", "other"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "server"); + fs_add_null(fs, "location"); + fs_add_null(fs, "usn"); + fs_add_null(fs, "st"); + fs_add_null(fs, "ext"); + fs_add_null(fs, "cache_control"); + fs_add_null(fs, "x_user_agent"); + fs_add_null(fs, "agent"); + fs_add_null(fs, "date"); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); + fs_add_null(fs, "data"); + } +} + +static fielddef_t fields[] = { + {.name = "classification", + .type = "string", + .desc = "packet classification"}, + {.name = "success", + .type = "bool", + .desc = "is response considered success"}, + + {.name = "server", .type = "string", .desc = "UPnP server"}, + {.name = "location", .type = "string", .desc = "UPnP location"}, + {.name = "usn", .type = "string", .desc = "UPnP usn"}, + {.name = "st", .type = "string", .desc = "UPnP st"}, + {.name = "ext", .type = "string", .desc = "UPnP ext"}, + {.name = "cache_control", .type = "string", .desc = "UPnP cache-control"}, + {.name = "x_user_agent", .type = "string", .desc = "UPnP x-user-agent"}, + {.name = "agent", .type = "string", .desc = "UPnP agent"}, + {.name = "date", .type = "string", .desc = "UPnP date"}, + + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + ICMP_FIELDSET_FIELDS, + {.name = "data", .type = "binary", .desc = "UDP payload"}}; + +probe_module_t module_upnp = { + .name = "upnp", + .max_packet_length = 139, + .pcap_filter = "udp || icmp", + .pcap_snaplen = 2048, + .port_args = 1, + .global_initialize = &upnp_global_initialize, + .thread_initialize = &upnp_init_perthread, + .make_packet = &udp_make_packet, + .print_packet = &udp_print_packet, + .process_packet = &upnp_process_packet, + .validate_packet = &upnp_validate_packet, + // UPnP isn't actually dynamic, however, we don't handle escaping + // properly in the CSV module and this will force users to use JSON. + .output_type = OUTPUT_TYPE_DYNAMIC, + .close = NULL, + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response.", + .fields = fields, + .numfields = 18}; diff --git a/src/probe_modules/packet.c b/src/probe_modules/packet.c new file mode 100644 index 0000000..f93f425 --- /dev/null +++ b/src/probe_modules/packet.c @@ -0,0 +1,252 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" +#include "packet.h" + +#include "../state.h" + +#ifndef NDEBUG +void print_macaddr(struct ifreq *i) +{ + printf("Device %s -> Ethernet %02x:%02x:%02x:%02x:%02x:%02x\n", + i->ifr_name, (int)((unsigned char *)&i->ifr_addr.sa_data)[0], + (int)((unsigned char *)&i->ifr_addr.sa_data)[1], + (int)((unsigned char *)&i->ifr_addr.sa_data)[2], + (int)((unsigned char *)&i->ifr_addr.sa_data)[3], + (int)((unsigned char *)&i->ifr_addr.sa_data)[4], + (int)((unsigned char *)&i->ifr_addr.sa_data)[5]); +} +#endif /* NDEBUG */ + +#define IP_ADDR_LEN_STR 20 + +void fprintf_ip_header(FILE *fp, struct ip *iph) +{ + struct in_addr *s = (struct in_addr *)&(iph->ip_src); + struct in_addr *d = (struct in_addr *)&(iph->ip_dst); + + char srcip[IP_ADDR_LEN_STR + 1]; + char dstip[IP_ADDR_LEN_STR + 1]; + // inet_ntoa is a const char * so we if just call it in + // fprintf, you'll get back wrong results since we're + // calling it twice. + strncpy(srcip, inet_ntoa(*s), IP_ADDR_LEN_STR - 1); + strncpy(dstip, inet_ntoa(*d), IP_ADDR_LEN_STR - 1); + + srcip[IP_ADDR_LEN_STR] = '\0'; + dstip[IP_ADDR_LEN_STR] = '\0'; + + fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %#04X }\n", srcip, + dstip, ntohs(iph->ip_sum)); +} + +void fprintf_eth_header(FILE *fp, struct ether_header *ethh) +{ + if (!zconf.send_ip_pkts) { + fprintf(fp, + "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | " + "dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n", + (int)((unsigned char *)ethh->ether_shost)[0], + (int)((unsigned char *)ethh->ether_shost)[1], + (int)((unsigned char *)ethh->ether_shost)[2], + (int)((unsigned char *)ethh->ether_shost)[3], + (int)((unsigned char *)ethh->ether_shost)[4], + (int)((unsigned char *)ethh->ether_shost)[5], + (int)((unsigned char *)ethh->ether_dhost)[0], + (int)((unsigned char *)ethh->ether_dhost)[1], + (int)((unsigned char *)ethh->ether_dhost)[2], + (int)((unsigned char *)ethh->ether_dhost)[3], + (int)((unsigned char *)ethh->ether_dhost)[4], + (int)((unsigned char *)ethh->ether_dhost)[5]); + } +} + +void make_eth_header(struct ether_header *ethh, macaddr_t *src, macaddr_t *dst) +{ + memcpy(ethh->ether_shost, src, ETHER_ADDR_LEN); + memcpy(ethh->ether_dhost, dst, ETHER_ADDR_LEN); + ethh->ether_type = htons(ETHERTYPE_IP); +} + +void make_ip_header(struct ip *iph, uint8_t protocol, uint16_t len) +{ + iph->ip_hl = 5; // Internet Header Length + iph->ip_v = 4; // IPv4 + iph->ip_tos = 0; // Type of Service + iph->ip_len = len; + iph->ip_id = htons(31337); // identification number + iph->ip_off = 0; // fragmentation flag + iph->ip_ttl = MAXTTL; // time to live (TTL) + iph->ip_p = protocol; // upper layer protocol => TCP + // we set the checksum = 0 for now because that's + // what it needs to be when we run the IP checksum + iph->ip_sum = 0; +} + +void make_icmp_header(struct icmp *buf) +{ + memset(buf, 0, sizeof(struct icmp)); + buf->icmp_type = ICMP_ECHO; + buf->icmp_code = 0; + buf->icmp_seq = 0; +} + +void make_tcp_header(struct tcphdr *tcp_header, uint16_t th_flags) +{ + tcp_header->th_seq = random(); + tcp_header->th_ack = 0; + tcp_header->th_x2 = 0; + tcp_header->th_off = 5; // data offset + tcp_header->th_flags = 0; + tcp_header->th_flags |= th_flags; + tcp_header->th_win = htons(64240); // largest possible window + tcp_header->th_sum = 0; + tcp_header->th_urp = 0; +} + +size_t set_mss_option(struct tcphdr *tcp_header) +{ + // This only sets MSS, which is a single-word option. + size_t header_size = tcp_header->th_off * 4; + uint8_t *base = (uint8_t *)tcp_header; + uint8_t *last_opt = (uint8_t *)base + header_size; + + // TCP Option "header" + last_opt[0] = 2; // MSS + last_opt[1] = 4; // MSS is 4 bytes long + + // Default Linux MSS is 1460, which 0x05b4 + last_opt[2] = 0x05; + last_opt[3] = 0xb4; + + tcp_header->th_off += 1; + return tcp_header->th_off * 4; +} + +void make_udp_header(struct udphdr *udp_header, uint16_t len) +{ + udp_header->uh_ulen = htons(len); + // checksum ignored in IPv4 if 0 + udp_header->uh_sum = 0; +} + +int icmp_helper_validate(const struct ip *ip_hdr, uint32_t len, + size_t min_l4_len, struct ip **probe_pkt, + size_t *probe_len) +{ + // We're only equipped to handle ICMP packets at this point + assert(ip_hdr->ip_p == IPPROTO_ICMP); + + // Several ICMP responses can be generated by hosts along the way in + // response to a non-ICMP probe packet. These include: + // * Source quench (ICMP_SOURCE_QUENCH) + // * Destination Unreachable (ICMP_DEST_UNREACH) + // * Redirect (ICMP_REDIRECT) + // * Time exceeded (ICMP_TIME_EXCEEDED) + // In all of these cases, the IP header and first 8 bytes of the + // original packet are included in the responses and can be used + // to understand where the probe packet was sent. + + // Check if the response was large enough to contain an IP header + const uint32_t min_len = 4 * ip_hdr->ip_hl + ICMP_HEADER_SIZE + + sizeof(struct ip) + min_l4_len; + if (len < min_len) { + return PACKET_INVALID; + } + // Check that ICMP response is one of these four + struct icmp *icmp = (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + if (!(icmp->icmp_type == ICMP_UNREACH || + icmp->icmp_type == ICMP_SOURCEQUENCH || + icmp->icmp_type == ICMP_REDIRECT || + icmp->icmp_type == ICMP_TIMXCEED)) { + return PACKET_INVALID; + } + struct ip *ip_inner = (struct ip *)((char *)icmp + ICMP_HEADER_SIZE); + size_t inner_packet_len = len - (4 * ip_hdr->ip_hl + ICMP_HEADER_SIZE); + // Now we know the actual inner ip length, we should recheck the buffer + // to make sure it has enough room for the application layer packet + if (inner_packet_len < (4 * ip_inner->ip_hl + min_l4_len)) { + return PACKET_INVALID; + } + // find original destination IP and check that we sent a packet + // to that IP address + uint32_t dest = ip_inner->ip_dst.s_addr; + if (!blocklist_is_allowed(dest)) { + return PACKET_INVALID; + } + *probe_pkt = ip_inner; + *probe_len = inner_packet_len; + return PACKET_VALID; +} + +void fs_add_null_icmp(fieldset_t *fs) +{ + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); +} + +void fs_add_failure_no_port(fieldset_t *fs) +{ + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); +} + +void fs_populate_icmp_from_iphdr(struct ip *ip, size_t len, fieldset_t *fs) +{ + assert(ip && "no ip header provide to fs_populate_icmp_from_iphdr"); + assert(fs && "no fieldset provided to fs_populate_icmp_from_iphdr"); + struct icmp *icmp = get_icmp_header(ip, len); + assert(icmp); + // ICMP unreach comes from another server (not the one we sent a + // probe to); But we will fix up saddr to be who we sent the + // probe to, in case you care. + struct ip *ip_inner = get_inner_ip_header(icmp, len); + fs_modify_string(fs, "saddr", make_ip_str(ip_inner->ip_dst.s_addr), 1); + // Add other ICMP fields from within the header + fs_add_string(fs, "icmp_responder", make_ip_str(ip->ip_src.s_addr), 1); + fs_add_uint64(fs, "icmp_type", icmp->icmp_type); + fs_add_uint64(fs, "icmp_code", icmp->icmp_code); + if (icmp->icmp_code <= ICMP_UNREACH_PRECEDENCE_CUTOFF) { + fs_add_constchar(fs, "icmp_unreach_str", + icmp_unreach_strings[icmp->icmp_code]); + } else { + fs_add_constchar(fs, "icmp_unreach_str", "unknown"); + } +} + +// Note: caller must free return value +char *make_ip_str(uint32_t ip) +{ + struct in_addr t; + t.s_addr = ip; + const char *temp = inet_ntoa(t); + char *retv = xmalloc(strlen(temp) + 1); + strcpy(retv, temp); + return retv; +} + +const char *icmp_unreach_strings[] = { + "network unreachable", "host unreachable", + "protocol unreachable", "port unreachable", + "fragments required", "source route failed", + "network unknown", "host unknown", + "source host isolated", "network admin. prohibited", + "host admin. prohibited", "network unreachable TOS", + "host unreachable TOS", "communication admin. prohibited", + "host presdence violation", "precedence cutoff"}; diff --git a/src/probe_modules/packet.h b/src/probe_modules/packet.h new file mode 100644 index 0000000..5b78ab3 --- /dev/null +++ b/src/probe_modules/packet.h @@ -0,0 +1,243 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../../lib/includes.h" +#include "../../lib/blocklist.h" +#include "../../lib/pbm.h" +#include "../state.h" + +#ifndef PACKET_H +#define PACKET_H + +#define MAX_PACKET_SIZE 4096 + +#define ICMP_UNREACH_HEADER_SIZE 8 + +#define PACKET_VALID 1 +#define PACKET_INVALID 0 + +#define ICMP_HEADER_SIZE 8 + +#define PRINT_PACKET_SEP \ + "------------------------------------------------------\n" + +#define CLASSIFICATION_SUCCESS_FIELDSET_FIELDS \ + {.name = "classification", \ + .type = "string", \ + .desc = "packet classification"}, \ + { \ + .name = "success", .type = "bool", \ + .desc = "is response considered success" \ + } + +#define CLASSIFICATION_SUCCESS_FIELDSET_LEN 2 + +#define ICMP_FIELDSET_FIELDS \ + {.name = "icmp_responder", \ + .type = "string", \ + .desc = "Source IP of ICMP_UNREACH messages"}, \ + {.name = "icmp_type", .type = "int", .desc = "icmp message type"}, \ + {.name = "icmp_code", \ + .type = "int", \ + .desc = "icmp message sub type code"}, \ + { \ + .name = "icmp_unreach_str", .type = "string", \ + .desc = \ + "for icmp_unreach responses, the string version of icmp_code (e.g. network-unreach)" \ + } + +#define ICMP_FIELDSET_LEN 4 + +typedef unsigned short __attribute__((__may_alias__)) alias_unsigned_short; + +void make_eth_header(struct ether_header *ethh, macaddr_t *src, macaddr_t *dst); + +void make_ip_header(struct ip *iph, uint8_t, uint16_t); +void make_tcp_header(struct tcphdr *, uint16_t th_flags); +size_t set_mss_option(struct tcphdr *tcp_header); +void make_icmp_header(struct icmp *); +void make_udp_header(struct udphdr *udp_header, uint16_t len); +void fprintf_ip_header(FILE *fp, struct ip *iph); +void fprintf_eth_header(FILE *fp, struct ether_header *ethh); + +static inline unsigned short in_checksum(unsigned short *ip_pkt, int len) +{ + unsigned long sum = 0; + for (int nwords = len / 2; nwords > 0; nwords--) { + sum += *ip_pkt++; + } + if (len % 2 == 1) { + sum += *((unsigned char *)ip_pkt); + } + sum = (sum >> 16) + (sum & 0xffff); + return (unsigned short)(~sum); +} + +static inline unsigned short in_icmp_checksum(unsigned short *ip_pkt, int len) +{ + unsigned long sum = 0; + for (int nwords = len / 2; nwords > 0; nwords--) { + sum += *ip_pkt++; + } + if (len % 2 == 1) { + sum += *((unsigned char *)ip_pkt); + } + sum = (sum >> 16) + (sum & 0xffff); + return (unsigned short)(~sum); +} + +static inline unsigned short zmap_ip_checksum(unsigned short *buf) +{ + return in_checksum(buf, (int)sizeof(struct ip)); +} + +static inline unsigned short icmp_checksum(unsigned short *buf, size_t buflen) +{ + return in_icmp_checksum(buf, buflen); +} + +static inline uint16_t tcp_checksum(unsigned short len_tcp, uint32_t saddr, + uint32_t daddr, struct tcphdr *tcp_pkt) +{ + alias_unsigned_short *src_addr = (alias_unsigned_short *)&saddr; + alias_unsigned_short *dest_addr = (alias_unsigned_short *)&daddr; + + unsigned char prot_tcp = 6; + unsigned long sum = 0; + int nleft = len_tcp; + unsigned short *w; + + w = (unsigned short *)tcp_pkt; + // calculate the checksum for the tcp header and tcp data + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + // if nleft is 1 there is still one byte left. + // We add a padding byte (0xFF) to build a 16bit word + if (nleft > 0) { + sum += *w & ntohs(0xFF00); + } + // add the pseudo header + sum += src_addr[0]; + sum += src_addr[1]; + sum += dest_addr[0]; + sum += dest_addr[1]; + sum += htons(len_tcp); + sum += htons(prot_tcp); + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + // Take the one's complement of sum + return (unsigned short)(~sum); +} + +// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise +static inline int check_dst_port(uint16_t port, int num_ports, + uint32_t *validation) +{ + if (port > zconf.source_port_last || port < zconf.source_port_first) { + return 0; + } + int32_t to_validate = port - zconf.source_port_first; + int32_t min = validation[1] % num_ports; + int32_t max = (validation[1] + zconf.packet_streams - 1) % num_ports; + + if (min <= max) { + return (to_validate <= max && to_validate >= min); + } else { + return (to_validate <= max ^ to_validate >= min); + } +} + +static inline uint16_t get_src_port(int num_ports, int probe_num, + uint32_t *validation) +{ + return zconf.source_port_first + + ((validation[1] + probe_num) % num_ports); +} + +static inline int check_src_port(uint16_t port, const struct port_conf *ports) +{ + return bm_check(ports->port_bitmap, port); +} + +static inline struct ip *get_ip_header(const u_char *packet, uint32_t len) +{ + if (len < sizeof(struct ether_header)) { + return NULL; + } + return (struct ip *)&packet[sizeof(struct ether_header)]; +} + +static inline struct tcphdr *get_tcp_header(const struct ip *ip_hdr, + uint32_t len) +{ + // buf not large enough to contain expected udp header + if ((4 * ip_hdr->ip_hl + sizeof(struct tcphdr)) > len) { + return NULL; + } + return (struct tcphdr *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); +} + +static inline struct udphdr *get_udp_header(const struct ip *ip_hdr, + uint32_t len) +{ + // buf not large enough to contain expected udp header + if ((4 * ip_hdr->ip_hl + sizeof(struct udphdr)) > len) { + return NULL; + } + return (struct udphdr *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); +} + +static inline struct icmp *get_icmp_header(const struct ip *ip_hdr, + uint32_t len) +{ + // buf not large enough to contain expected udp header + if ((4 * ip_hdr->ip_hl + sizeof(struct icmp)) > len) { + return NULL; + } + return (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); +} + +static inline uint8_t *get_udp_payload(const struct udphdr *udp, + UNUSED uint32_t len) +{ + return (uint8_t *)(&udp[1]); +} + +static inline struct ip *get_inner_ip_header(const struct icmp *icmp, + uint32_t len) +{ + if (len < (ICMP_UNREACH_HEADER_SIZE + sizeof(struct ip))) { + return NULL; + } + return (struct ip *)((char *)icmp + ICMP_UNREACH_HEADER_SIZE); +} + +// Note: caller must free return value +char *make_ip_str(uint32_t ip); + +extern const char *icmp_unreach_strings[]; + +int icmp_helper_validate(const struct ip *ip_hdr, uint32_t len, + size_t min_l4_len, struct ip **probe_pkt, + size_t *probe_len); + +void fs_add_null_icmp(fieldset_t *fs); + +void fs_populate_icmp_from_iphdr(struct ip *ip, size_t len, fieldset_t *fs); + +#endif diff --git a/src/probe_modules/probe_modules.c b/src/probe_modules/probe_modules.c new file mode 100644 index 0000000..acada5d --- /dev/null +++ b/src/probe_modules/probe_modules.c @@ -0,0 +1,130 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../../lib/logger.h" +#include "../../lib/xalloc.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" + +extern probe_module_t module_tcp_synscan; +extern probe_module_t module_tcp_synackscan; +extern probe_module_t module_icmp_echo; +extern probe_module_t module_icmp_echo_time; +extern probe_module_t module_udp; +extern probe_module_t module_ntp; +extern probe_module_t module_upnp; +extern probe_module_t module_dns; +extern probe_module_t module_bacnet; +// ADD YOUR MODULE HERE + +probe_module_t *probe_modules[] = { + &module_tcp_synscan, &module_tcp_synackscan, + &module_icmp_echo, &module_icmp_echo_time, + &module_udp, &module_ntp, + &module_upnp, &module_dns, + &module_bacnet + // ADD YOUR MODULE HERE +}; + +probe_module_t *get_probe_module_by_name(const char *name) +{ + int len = (int)(sizeof(probe_modules) / sizeof(probe_modules[0])); + for (int i = 0; i < len; i++) { + if (!strcmp(probe_modules[i]->name, name)) { + return probe_modules[i]; + } + } + return NULL; +} + +void print_probe_modules(void) +{ + int len = (int)(sizeof(probe_modules) / sizeof(probe_modules[0])); + for (int i = 0; i < len; i++) { + printf("%s\n", probe_modules[i]->name); + } +} + +void fs_add_ip_fields(fieldset_t *fs, struct ip *ip) +{ + // WARNING: you must update fs_ip_fields_len as well + // as the definitions set (ip_fiels) if you + // change the fields added below: + fs_add_string(fs, "saddr", make_ip_str(ip->ip_src.s_addr), 1); + fs_add_uint64(fs, "saddr_raw", (uint64_t)ip->ip_src.s_addr); + fs_add_string(fs, "daddr", make_ip_str(ip->ip_dst.s_addr), 1); + fs_add_uint64(fs, "daddr_raw", (uint64_t)ip->ip_dst.s_addr); + fs_add_uint64(fs, "ipid", ntohs(ip->ip_id)); + fs_add_uint64(fs, "ttl", ip->ip_ttl); +} + +#define TIMESTR_LEN 55 + +void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown) +{ + fs_add_bool(fs, "repeat", is_repeat); + fs_add_bool(fs, "cooldown", in_cooldown); + + char *timestr = xmalloc(TIMESTR_LEN + 1); + char *timestr_ms = xmalloc(TIMESTR_LEN + 1); + struct timeval t; + gettimeofday(&t, NULL); + struct tm *ptm = localtime(&t.tv_sec); + strftime(timestr, TIMESTR_LEN, "%Y-%m-%dT%H:%M:%S.%%03d%z", ptm); + snprintf(timestr_ms, TIMESTR_LEN, timestr, t.tv_usec / 1000); + free(timestr); + fs_add_string(fs, "timestamp_str", timestr_ms, 1); + fs_add_uint64(fs, "timestamp_ts", (uint64_t)t.tv_sec); + fs_add_uint64(fs, "timestamp_us", (uint64_t)t.tv_usec); +} + +int ip_fields_len = 6; +fielddef_t ip_fields[] = { + {.name = "saddr", + .type = "string", + .desc = "source IP address of response"}, + {.name = "saddr_raw", + .type = "int", + .desc = "network order integer form of source IP address"}, + {.name = "daddr", + .type = "string", + .desc = "destination IP address of response"}, + {.name = "daddr_raw", + .type = "int", + .desc = "network order integer form of destination IP address"}, + {.name = "ipid", + .type = "int", + .desc = "IP identification number of response"}, + {.name = "ttl", .type = "int", .desc = "time-to-live of response packet"}}; + +int sys_fields_len = 5; +fielddef_t sys_fields[] = { + {.name = "repeat", + .type = "bool", + .desc = "Is response a repeat response from host"}, + {.name = "cooldown", + .type = "bool", + .desc = "Was response received during the cooldown period"}, + {.name = "timestamp_str", + .type = "string", + .desc = "timestamp of when response arrived in ISO8601 format."}, + {.name = "timestamp_ts", + .type = "int", + .desc = "timestamp of when response arrived in seconds since Epoch"}, + {.name = "timestamp_us", + .type = "int", + .desc = + "microsecond part of timestamp (e.g. microseconds since 'timestamp-ts')"}}; diff --git a/src/probe_modules/probe_modules.h b/src/probe_modules/probe_modules.h new file mode 100644 index 0000000..776c582 --- /dev/null +++ b/src/probe_modules/probe_modules.h @@ -0,0 +1,110 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../state.h" +#include "../fieldset.h" + +#ifndef PROBE_MODULES_H +#define PROBE_MODULES_H + +#define OUTPUT_TYPE_STATIC 1 +#define OUTPUT_TYPE_DYNAMIC 2 + +typedef struct probe_response_type { + const uint8_t is_success; + const char *name; +} response_type_t; + +typedef int (*probe_global_init_cb)(struct state_conf *); + +typedef int (*probe_thread_init_cb)(void *packetbuf, macaddr_t *src_mac, + macaddr_t *gw_mac, void **arg_ptr); + +// The make_packet callback is passed a buffer pointing at an ethernet header. +// The buffer is MAX_PACKET_SIZE bytes. The callback must update the value +// pointed at by buf_len with the actual length of the packet. The contents of +// the buffer will match the previous packet sent. Every invocation of +// make_packet contains a unique (src_ip, probe_num) tuple. +// +// The probe module is responsible for populating the IP header. The src_ip, +// dst_ip, and ttl are provided by the framework and must be set on the IP +// header. +// +// The uin32_t validation parameter is a pointer to four 4-byte words of +// validation data. The data is deterministic based on the the validation +// state, and is constant across a src_ip. To get the src_port, use the +// get_src_port function which takes probe_num and validation as parameters. +// +typedef int (*probe_make_packet_cb)(void *packetbuf, size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, + port_n_t dst_port, uint8_t ttl, + uint32_t *validation, int probe_num, + void *arg); + +typedef void (*probe_print_packet_cb)(FILE *, void *packetbuf); + +typedef int (*probe_close_cb)(struct state_conf *, struct state_send *, + struct state_recv *); + +typedef int (*probe_validate_packet_cb)(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation, + const struct port_conf *ports); + +typedef void (*probe_classify_packet_cb)(const u_char *packetbuf, uint32_t len, + fieldset_t *, uint32_t *validation, + const struct timespec ts); + +typedef struct probe_module { + const char *name; + + // TODO(dadrian): Completely get rid of this. We can do bandwidth rate + // limiting by actually counting how much data is sent over the wire. We + // know the lengths of packets from the make_packet API. + size_t max_packet_length; + + const char *pcap_filter; + size_t pcap_snaplen; + + // Should ZMap complain if the user hasn't specified valid + // source and target port numbers? + uint8_t port_args; + + probe_global_init_cb global_initialize; + probe_thread_init_cb thread_initialize; + probe_make_packet_cb make_packet; + probe_print_packet_cb print_packet; + probe_validate_packet_cb validate_packet; + probe_classify_packet_cb process_packet; + probe_close_cb close; + int output_type; + fielddef_t *fields; + int numfields; + const char *helptext; + +} probe_module_t; + +probe_module_t *get_probe_module_by_name(const char *); + +void fs_add_ip_fields(fieldset_t *fs, struct ip *ip); +void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown); +void print_probe_modules(void); + +extern int ip_fields_len; +extern int sys_fields_len; +extern fielddef_t ip_fields[]; +extern fielddef_t sys_fields[]; + +#endif // HEADER_PROBE_MODULES_H diff --git a/src/recv-internal.h b/src/recv-internal.h new file mode 100644 index 0000000..425d5f2 --- /dev/null +++ b/src/recv-internal.h @@ -0,0 +1,20 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_RECV_INTERNAL_H +#define ZMAP_RECV_INTERNAL_H + +#include + +void handle_packet(uint32_t buflen, const uint8_t *bytes, + const struct timespec ts); +void recv_init(void); +void recv_packets(void); +void recv_cleanup(void); + +#endif /* ZMAP_RECV_INTERNAL_H */ diff --git a/src/recv-pcap.c b/src/recv-pcap.c new file mode 100644 index 0000000..9c3972b --- /dev/null +++ b/src/recv-pcap.c @@ -0,0 +1,164 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv.h" + +#include +#include +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" + +#include +#include +#if defined __linux__ && __linux__ +#include +#endif + +#include "recv-internal.h" +#include "state.h" + +#include "probe_modules/probe_modules.h" + +#define PCAP_PROMISC 1 +#define PCAP_TIMEOUT 1000 + +static pcap_t *pc = NULL; + +void packet_cb(u_char __attribute__((__unused__)) * user, + const struct pcap_pkthdr *p, const u_char *bytes) +{ + struct timespec ts; + if (!p) { + return; + } + if (zrecv.filter_success >= zconf.max_results) { + // Libpcap can process multiple packets per pcap_dispatch; + // we need to throw out results once we've + // gotten our --max-results worth. + return; + } + + // length of entire packet captured by libpcap + uint32_t buflen = (uint32_t)p->caplen; + ts.tv_sec = p->ts.tv_sec; + ts.tv_nsec = p->ts.tv_usec * 1000; + handle_packet(buflen, bytes, ts); +} + +#define BPFLEN 1024 + +void recv_init(void) +{ + char bpftmp[BPFLEN]; + char errbuf[PCAP_ERRBUF_SIZE]; + + pc = pcap_open_live(zconf.iface, zconf.probe_module->pcap_snaplen, + PCAP_PROMISC, PCAP_TIMEOUT, errbuf); + if (pc == NULL) { + log_fatal("recv", "could not open device %s: %s", zconf.iface, + errbuf); + } + switch (pcap_datalink(pc)) { + case DLT_NULL: + // utun on macOS + log_debug("recv", "BSD loopback encapsulation"); + zconf.data_link_size = 4; + break; + case DLT_EN10MB: + log_debug("recv", "Data link layer Ethernet"); + zconf.data_link_size = sizeof(struct ether_header); + break; + case DLT_RAW: + log_info("recv", "Data link RAW"); + zconf.data_link_size = 0; + break; +#if defined __linux__ && __linux__ + case DLT_LINUX_SLL: + log_info("recv", "Data link cooked socket"); + zconf.data_link_size = SLL_HDR_LEN; + break; +#endif + default: + log_error("recv", "unknown data link layer: %u", pcap_datalink(pc)); + } + + struct bpf_program bpf; + + if (!zconf.send_ip_pkts) { + snprintf(bpftmp, sizeof(bpftmp) - 1, + "not ether src %02x:%02x:%02x:%02x:%02x:%02x", + zconf.hw_mac[0], zconf.hw_mac[1], zconf.hw_mac[2], + zconf.hw_mac[3], zconf.hw_mac[4], zconf.hw_mac[5]); + assert(strlen(zconf.probe_module->pcap_filter) + 10 < + (BPFLEN - strlen(bpftmp))); + } else { + bpftmp[0] = 0; + } + if (zconf.probe_module->pcap_filter) { + if (!zconf.send_ip_pkts) { + strcat(bpftmp, " and ("); + } else { + strcat(bpftmp, "("); + } + strcat(bpftmp, zconf.probe_module->pcap_filter); + strcat(bpftmp, ")"); + } + if (strcmp(bpftmp, "")) { + if (pcap_compile(pc, &bpf, bpftmp, 1, 0) < 0) { + log_fatal("recv", "couldn't compile filter"); + } + if (pcap_setfilter(pc, &bpf) < 0) { + log_fatal("recv", "couldn't install filter"); + } + } + // set pcap_dispatch to not hang if it never receives any packets + // this could occur if you ever scan a small number of hosts as + // documented in issue #74. + if (pcap_setnonblock(pc, 1, errbuf) == -1) { + log_fatal("recv", "pcap_setnonblock error:%s", errbuf); + } +} + +void recv_packets(void) +{ + int ret = pcap_dispatch(pc, -1, packet_cb, NULL); + if (ret == -1) { + log_fatal("recv", "pcap_dispatch error"); + } else if (ret == 0) { + usleep(1000); + } +} + +void recv_cleanup(void) +{ + pcap_close(pc); + pc = NULL; +} + +int recv_update_stats(void) +{ + if (!pc) { + return EXIT_FAILURE; + } + struct pcap_stat pcst; + if (pcap_stats(pc, &pcst)) { + log_error("recv", "unable to retrieve pcap statistics: %s", + pcap_geterr(pc)); + return EXIT_FAILURE; + } else { + zrecv.pcap_recv = pcst.ps_recv; + zrecv.pcap_drop = pcst.ps_drop; + zrecv.pcap_ifdrop = pcst.ps_ifdrop; + } + return EXIT_SUCCESS; +} diff --git a/src/recv-pfring.c b/src/recv-pfring.c new file mode 100644 index 0000000..dbc4742 --- /dev/null +++ b/src/recv-pfring.c @@ -0,0 +1,82 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv.h" +#include "recv-internal.h" + +#include "../lib/includes.h" +#include "../lib/logger.h" + +#include +#include + +#include + +#include "state.h" + +static pfring_zc_pkt_buff *pf_buffer; +static pfring_zc_queue *pf_recv; + +void recv_init() +{ + // Get the socket and packet handle + pf_recv = zconf.pf.recv; + pf_buffer = pfring_zc_get_packet_handle(zconf.pf.cluster); + if (pf_buffer == NULL) { + log_fatal("recv", "Could not get packet handle: %s", + strerror(errno)); + } +} + +void recv_cleanup() +{ + if (!pf_recv) { + return; + } + pfring_zc_sync_queue(pf_recv, rx_only); +} + +void recv_packets() +{ + int ret; + // Poll for packets + do { + ret = pfring_zc_recv_pkt(pf_recv, &pf_buffer, 0); + if (ret == 0) { + usleep(1000); + } + } while (ret == 0); + // Handle other errors, by not doing anything and logging + if (ret != 1) { + log_error("recv", "Error: %d", ret); + return; + } + // Successfully got a packet, now handle it + struct timespec ts; + ts.tv_sec = pf_buffer->ts.tv_sec; + ts.tv_nsec = pf_buffer->ts.tv_nsec; //* 1000; + + uint8_t *pkt_buf = pfring_zc_pkt_buff_data(pf_buffer, pf_recv); + handle_packet(pf_buffer->len, pkt_buf, ts); +} + +int recv_update_stats(void) +{ + if (!pf_recv) { + return EXIT_FAILURE; + } + pfring_zc_stat pfst; + if (pfring_zc_stats(pf_recv, &pfst)) { + log_error("recv", "unable to retrieve pfring statistics"); + return EXIT_FAILURE; + } else { + zrecv.pcap_recv = pfst.recv; + zrecv.pcap_drop = pfst.drop; + } + return EXIT_SUCCESS; +} diff --git a/src/recv.c b/src/recv.c new file mode 100644 index 0000000..691345b --- /dev/null +++ b/src/recv.c @@ -0,0 +1,238 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv.h" + +#include + +#include "../lib/includes.h" +#include "cachehash.h" +#include "../lib/util.h" +#include "../lib/logger.h" +#include "../lib/pbm.h" + +#include +#include + +#include "recv-internal.h" +#include "state.h" +#include "validate.h" +#include "fieldset.h" +#include "shard.h" +#include "expression.h" +#include "probe_modules/packet.h" +#include "probe_modules/probe_modules.h" +#include "output_modules/output_modules.h" + +static u_char fake_eth_hdr[65535]; +// bitmap of observed IP addresses +static uint8_t **seen = NULL; +static cachehash *ch = NULL; + +void handle_packet(uint32_t buflen, const u_char *bytes, + const struct timespec ts) +{ + if ((sizeof(struct ip) + zconf.data_link_size) > buflen) { + // buffer not large enough to contain ethernet + // and ip headers. further action would overrun buf + return; + } + struct ip *ip_hdr = (struct ip *)&bytes[zconf.data_link_size]; + uint32_t src_ip = ip_hdr->ip_src.s_addr; + uint16_t src_port = 0; + + uint32_t len_ip_and_payload = + buflen - (zconf.send_ip_pkts ? 0 : sizeof(struct ether_header)); + // extract port if TCP or UDP packet to both generate validation data and to + // check if the response is a duplicate + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len_ip_and_payload); + if (tcp) { + src_port = tcp->th_sport; + } + } else if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len_ip_and_payload); + if (udp) { + src_port = udp->uh_sport; + } + } + + uint32_t validation[VALIDATE_BYTES / sizeof(uint8_t)]; + // TODO: for TTL exceeded messages, ip_hdr->saddr is going to be + // different and we must calculate off potential payload message instead + validate_gen(ip_hdr->ip_dst.s_addr, ip_hdr->ip_src.s_addr, src_port, + (uint8_t *)validation); + + if (!zconf.probe_module->validate_packet( + ip_hdr, len_ip_and_payload, &src_ip, validation, zconf.ports)) { + zrecv.validation_failed++; + return; + } else { + zrecv.validation_passed++; + } + // woo! We've validated that the packet is a response to our scan + int is_repeat = 0; + if (zconf.dedup_method == DEDUP_METHOD_FULL) { + is_repeat = pbm_check(seen, ntohl(src_ip)); + } else if (zconf.dedup_method == DEDUP_METHOD_WINDOW) { + target_t t = {.ip = src_ip, .port = src_port, .status = 0}; + if (cachehash_get(ch, &t, sizeof(target_t))) { + is_repeat = 1; + } else { + cachehash_put(ch, &t, sizeof(target_t), (void *)1); + } + } + // track whether this is the first packet in an IP fragment. + if (ip_hdr->ip_off & IP_MF) { + zrecv.ip_fragments++; + } + + fieldset_t *fs = fs_new_fieldset(&zconf.fsconf.defs); + fs_add_ip_fields(fs, ip_hdr); + // HACK: + // probe modules expect the full ethernet frame + // in process_packet. For VPN, we only get back an IP frame. + // Here, we fake an ethernet frame (which is initialized to + // have ETH_P_IP proto and 00s for dest/src). + if (zconf.send_ip_pkts) { + const static uint32_t available_space = sizeof(fake_eth_hdr) - sizeof(struct ether_header); + assert(buflen > (uint32_t)zconf.data_link_size); + buflen -= zconf.data_link_size; + if (buflen > available_space) { + buflen = available_space; + } + memcpy(&fake_eth_hdr[sizeof(struct ether_header)], + bytes + zconf.data_link_size, buflen); + bytes = fake_eth_hdr; + buflen += sizeof(struct ether_header); + } + zconf.probe_module->process_packet(bytes, buflen, fs, validation, ts); + fs_add_system_fields(fs, is_repeat, zsend.complete); + int success_index = zconf.fsconf.success_index; + assert(success_index < fs->len); + int is_success = fs_get_uint64_by_index(fs, success_index); + + if (is_success) { + zrecv.success_total++; + if (!is_repeat) { + zrecv.success_unique++; + if (zconf.dedup_method == DEDUP_METHOD_FULL) { + pbm_set(seen, ntohl(src_ip)); + } else if (zconf.dedup_method == DEDUP_METHOD_WINDOW) { + } + } + if (zsend.complete) { + zrecv.cooldown_total++; + if (!is_repeat) { + zrecv.cooldown_unique++; + } + } + } else { + zrecv.failure_total++; + } + // probe module includes app_success field + if (zconf.fsconf.app_success_index >= 0) { + int is_app_success = + fs_get_uint64_by_index(fs, zconf.fsconf.app_success_index); + if (is_app_success) { + zrecv.app_success_total++; + if (!is_repeat) { + zrecv.app_success_unique++; + } + } + } + + fieldset_t *o = NULL; + // we need to translate the data provided by the probe module + // into a fieldset that can be used by the output module + if (!is_success && zconf.default_mode) { + goto cleanup; + } + if (is_repeat && zconf.default_mode) { + goto cleanup; + } + if (!evaluate_expression(zconf.filter.expression, fs)) { + goto cleanup; + } + zrecv.filter_success++; + o = translate_fieldset(fs, &zconf.fsconf.translation); + if (zconf.output_module && zconf.output_module->process_ip) { + zconf.output_module->process_ip(o); + } +cleanup: + fs_free(fs); + free(o); + if (zconf.output_module && zconf.output_module->update && + !(zrecv.success_unique % zconf.output_module->update_interval)) { + zconf.output_module->update(&zconf, &zsend, &zrecv); + } +} + +int recv_run(pthread_mutex_t *recv_ready_mutex) +{ + log_trace("recv", "recv thread started"); + log_debug("recv", "capturing responses on %s", zconf.iface); + if (!zconf.dryrun) { + recv_init(); + } + if (zconf.send_ip_pkts) { + struct ether_header *eth = (struct ether_header *)fake_eth_hdr; + memset(fake_eth_hdr, 0, sizeof(fake_eth_hdr)); + eth->ether_type = htons(ETHERTYPE_IP); + } + // initialize paged bitmap + if (zconf.dedup_method == DEDUP_METHOD_FULL) { + seen = pbm_init(); + } else if (zconf.dedup_method == DEDUP_METHOD_WINDOW) { + ch = cachehash_init(zconf.dedup_window_size, NULL); + } + if (zconf.default_mode) { + log_info("recv", + "duplicate responses will be excluded from output"); + log_info("recv", + "unsuccessful responses will be excluded from output"); + } else { + log_info( + "recv", + "duplicate responses will be passed to the output module"); + log_info( + "recv", + "unsuccessful responses will be passed to the output module"); + } + pthread_mutex_lock(recv_ready_mutex); + zconf.recv_ready = 1; + pthread_mutex_unlock(recv_ready_mutex); + zrecv.start = now(); + if (zconf.max_results == 0) { + zconf.max_results = -1; + } + + do { + if (zconf.dryrun) { + sleep(1); + } else { + recv_packets(); + if (zconf.max_results && + zrecv.filter_success >= zconf.max_results) { + break; + } + } + } while ( + !(zsend.complete && (now() - zsend.finish > zconf.cooldown_secs))); + zrecv.finish = now(); + // get final pcap statistics before closing + recv_update_stats(); + if (!zconf.dryrun) { + pthread_mutex_lock(recv_ready_mutex); + recv_cleanup(); + pthread_mutex_unlock(recv_ready_mutex); + } + zrecv.complete = 1; + log_debug("recv", "thread finished"); + return 0; +} diff --git a/src/recv.h b/src/recv.h new file mode 100644 index 0000000..07824f0 --- /dev/null +++ b/src/recv.h @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_RECV_H +#define ZMAP_RECV_H + +#include + +int recv_update_stats(void); +int recv_run(pthread_mutex_t *recv_ready_mutex); + +#endif /* ZMP_RECV_H */ diff --git a/src/send-bsd.h b/src/send-bsd.h new file mode 100644 index 0000000..46d8a2c --- /dev/null +++ b/src/send-bsd.h @@ -0,0 +1,115 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SEND_BSD_H +#define ZMAP_SEND_BSD_H + +#include +#include +#include +#include +#include + +#include "send.h" +#include "../lib/includes.h" + +#include +#include + +#ifdef ZMAP_SEND_LINUX_H +#error "Don't include both send-bsd.h and send-linux.h" +#endif + +int send_run_init(UNUSED sock_t sock) +{ + // Don't need to do anything on BSD-like variants + return EXIT_SUCCESS; +} + +int send_packet(sock_t sock, void *buf, int len, UNUSED uint32_t retry_ct) +{ + if (zconf.send_ip_pkts) { + struct ip *iph = (struct ip *)buf; + +#if defined(__APPLE__) || (defined(__FreeBSD__) && __FreeBSD_version < 1100030) + // Early BSD's raw IP sockets required IP headers to have len + // and off fields in host byte order, as they were being byte + // swapped on the way out. Getting byte order wrong would + // result in EINVAL from sendto(2) below. + // Most modern BSD systems have fixed this and removed the byte + // swapping on raw IP sockets, while some, notably macOS, still + // require header fields in host byte order. + // See ip(4) for details on byte order requirements of raw IP + // sockets. + // Swap byte order on the first send attempt for a given packet + // (retry_ct == 0), and rely on the caller to pass us the same + // buffer again for retries (retry_ct > 0), with the buffer + // still containing the header fields in corrected byte order. + if (retry_ct == 0) { + iph->ip_len = ntohs(iph->ip_len); + iph->ip_off = ntohs(iph->ip_off); + iph->ip_sum = 0; + } +#endif + + struct sockaddr_in sai; + bzero(&sai, sizeof(sai)); + sai.sin_family = AF_INET; + sai.sin_addr.s_addr = iph->ip_dst.s_addr; + return sendto(sock.sock, buf, len, 0, (struct sockaddr *)&sai, sizeof(sai)); + } else { + return write(sock.sock, buf, len); + } +} + +// macOS doesn't have sendmmsg as of Sonoma. Since we want a uniform interface, we'll emulate the send_batch used in Linux. +// FreeBSD does have sendmmsg, but it is a libc wrapper around the sendmsg syscall, without the perf benefits of sendmmsg. +// The behavior in sendmmsg is to send as many packets as possible until one fails, and then return the number of sent packets. +// Following the same pattern for consistency +// Returns - number of packets sent +// Returns -1 and sets errno if no packets could be sent successfully +int send_batch(sock_t sock, batch_t* batch, int retries) { + if (batch->len == 0) { + // nothing to send + return EXIT_SUCCESS; + } + int packets_sent = 0; + int rc = 0; + for (int packet_num = 0; packet_num < batch->len; packet_num++) { + for (int retry_ct = 0; retry_ct < retries; retry_ct++) { + rc = send_packet(sock, ((uint8_t *)batch->packets) + (packet_num * MAX_PACKET_SIZE), batch->lens[packet_num], retry_ct); + if (rc >= 0) { + packets_sent++; + break; + } + } + if (rc < 0) { + // packet couldn't be sent in retries number of attempts + struct in_addr addr; + addr.s_addr = batch->ips[packet_num]; + char addr_str_buf[INET_ADDRSTRLEN]; + const char *addr_str = + inet_ntop( + AF_INET, &addr, + addr_str_buf, + INET_ADDRSTRLEN); + if (addr_str != NULL) { + log_debug( "send-bsd", "send_packet failed for %s. %s", addr_str, + strerror( errno)); + } + } + } + if (packets_sent == 0) { + // simulating the return behaviour of the Linux send_mmsg sys call on error. Returns -1 and leaves + // errno as set by send_packet + return -1; + } + return packets_sent; +} + +#endif /* ZMAP_SEND_BSD_H */ diff --git a/src/send-linux.c b/src/send-linux.c new file mode 100644 index 0000000..4e9b80b --- /dev/null +++ b/src/send-linux.c @@ -0,0 +1,121 @@ +/* + * Copyright 2023 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "./send.h" +#include "./send-linux.h" + +int send_run_init(sock_t s) +{ + // Get the actual socket + int sock = s.sock; + // get source interface index + struct ifreq if_idx; + memset(&if_idx, 0, sizeof(struct ifreq)); + if (strlen(zconf.iface) >= IFNAMSIZ) { + log_error("send", "device interface name (%s) too long\n", + zconf.iface); + return EXIT_FAILURE; + } + strncpy(if_idx.ifr_name, zconf.iface, IFNAMSIZ - 1); + if (ioctl(sock, SIOCGIFINDEX, &if_idx) < 0) { + log_error("send", "%s", "SIOCGIFINDEX"); + return EXIT_FAILURE; + } + int ifindex = if_idx.ifr_ifindex; + + // destination address for the socket + memset((void *)&sockaddr, 0, sizeof(struct sockaddr_ll)); + sockaddr.sll_ifindex = ifindex; + sockaddr.sll_halen = ETH_ALEN; + if (zconf.send_ip_pkts) { + sockaddr.sll_protocol = htons(ETHERTYPE_IP); + } + memcpy(sockaddr.sll_addr, zconf.gw_mac, ETH_ALEN); + return EXIT_SUCCESS; +} + +int send_batch(sock_t sock, batch_t* batch, int retries) { + if (batch->len == 0) { + // nothing to send + return EXIT_SUCCESS; + } + struct mmsghdr msgvec [batch->capacity]; // Array of multiple msg header structures + struct msghdr msgs[batch->capacity]; + struct iovec iovs[batch->capacity]; + + for (int i = 0; i < batch->len; ++i) { + struct iovec *iov = &iovs[i]; + iov->iov_base = ((void *)batch->packets) + (i * MAX_PACKET_SIZE); + iov->iov_len = batch->lens[i]; + struct msghdr *msg = &msgs[i]; + memset(msg, 0, sizeof(struct msghdr)); + // based on https://github.com/torvalds/linux/blob/master/net/socket.c#L2180 + msg->msg_name = (struct sockaddr *)&sockaddr; + msg->msg_namelen = sizeof(struct sockaddr_ll); + msg->msg_iov = iov; + msg->msg_iovlen = 1; + msgvec[i].msg_hdr = *msg; + msgvec[i].msg_len = batch->lens[i]; + } + // set up per-retry variables, so we can only re-submit what didn't send successfully + struct mmsghdr* current_msg_vec = msgvec; + int total_packets_sent = 0; + int num_of_packets_in_batch = batch->len; + for (int i = 0; i < retries; i++) { + // according to manpages + // On success, sendmmsg() returns the number of messages sent from msgvec; if this is less than vlen, the + // caller can retry with a further sendmmsg() call to send the remaining messages. + // On error, -1 is returned, and errno is set to indicate the error. + int rv = sendmmsg(sock.sock, current_msg_vec, num_of_packets_in_batch, 0); + if (rv < 0) { + // retry if sending all packets failed + log_error("batch send", "error in sendmmsg: %s", strerror(errno)); + continue; + } + // if rv is positive, it gives the number of packets successfully sent + total_packets_sent += rv; + if (rv == num_of_packets_in_batch){ + // all packets in batch were sent successfully + break; + } + // batch send was only partially successful, we'll retry if we have retries available + log_warn("batch send", "only successfully sent %d packets out of a batch of %d packets", total_packets_sent, batch->len); + // per the manpages for sendmmsg, packets are sent sequentially and the call returns upon a + // failure, returning the number of packets successfully sent + // remove successfully sent packets from batch for retry + current_msg_vec = &msgvec[total_packets_sent]; + num_of_packets_in_batch = batch->len - total_packets_sent; + } + return total_packets_sent; +} diff --git a/src/send-linux.h b/src/send-linux.h new file mode 100644 index 0000000..7166cac --- /dev/null +++ b/src/send-linux.h @@ -0,0 +1,35 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ZMAP_SEND_LINUX_H +#define ZMAP_SEND_LINUX_H + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "./send.h" + + +#ifdef ZMAP_SEND_BSD_H +#error "Don't include both send-bsd.h and send-linux.h" +#endif + +// Dummy sockaddr for sendto +static struct sockaddr_ll sockaddr; + +int send_run_init(sock_t s); +int send_batch(sock_t sock, batch_t* batch, int retries); + +#endif /* ZMAP_SEND_LINUX_H */ diff --git a/src/send-pfring.h b/src/send-pfring.h new file mode 100644 index 0000000..9ef29b2 --- /dev/null +++ b/src/send-pfring.h @@ -0,0 +1,42 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SEND_PFRING_H +#define ZMAP_SEND_PFRING_H + +#include "../lib/includes.h" +#include + +#if defined(ZMAP_SEND_BSD_H) || defined(ZMAP_SEND_LINUX_H) +#error "Don't include send-bsd.h or send-linux.h with send-pfring.h" +#endif + +int send_run_init(sock_t socket) +{ + (void)socket; + + // All init for pfring happens in get_socket + return 0; +} + +int send_packet(sock_t sock, void *buf, int len, uint32_t idx) +{ + sock.pf.buffers[idx]->len = len; + memcpy(pfring_zc_pkt_buff_data(sock.pf.buffers[idx], sock.pf.queue), + buf, len); + int ret; + do { + ret = + pfring_zc_send_pkt(sock.pf.queue, &sock.pf.buffers[idx], 0); + } while (ret < 0); + return ret; +} + +void send_finish(sock_t sock) { pfring_zc_sync_queue(sock.pf.queue, tx_only); } + +#endif /* ZMAP_SEND_PFRING_H */ diff --git a/src/send.c b/src/send.c new file mode 100644 index 0000000..b6e43ec --- /dev/null +++ b/src/send.c @@ -0,0 +1,505 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "send.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/util.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/blocklist.h" +#include "../lib/lockfd.h" +#include "../lib/pbm.h" + +#include "aesrand.h" +#include "get_gateway.h" +#include "iterator.h" +#include "probe_modules/packet.h" +#include "probe_modules/probe_modules.h" +#include "shard.h" +#include "state.h" +#include "validate.h" + +// OS specific functions called by send_run +static inline int send_packet(sock_t sock, void *buf, int len, uint32_t idx); +static inline int send_batch(sock_t sock, batch_t *batch, int retries); +static inline int send_run_init(sock_t sock); + +// Include the right implementations +#if defined(PFRING) +#include "send-pfring.h" +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__DragonFly__) +#include "send-bsd.h" +#else /* LINUX */ +#include "send-linux.h" +#endif /* __APPLE__ || __FreeBSD__ || __NetBSD__ || __DragonFly__ */ + +// The iterator over the cyclic group + +// Lock for send run +static pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER; + +// Source ports for outgoing packets +static uint16_t num_src_ports; + +void sig_handler_increase_speed(UNUSED int signal) +{ + int old_rate = zconf.rate; + zconf.rate += (zconf.rate * 0.05); + log_info("send", "send rate increased from %i to %i pps.", old_rate, + zconf.rate); +} + +void sig_handler_decrease_speed(UNUSED int signal) +{ + int old_rate = zconf.rate; + zconf.rate -= (zconf.rate * 0.05); + log_info("send", "send rate decreased from %i to %i pps.", old_rate, + zconf.rate); +} + +// global sender initialize (not thread specific) +iterator_t *send_init(void) +{ + // generate a new primitive root and starting position + iterator_t *it; + uint32_t num_subshards = + (uint32_t)zconf.senders * (uint32_t)zconf.total_shards; + if (num_subshards > blocklist_count_allowed()) { + log_fatal("send", "senders * shards > allowed probes"); + } + if (zsend.max_targets && (num_subshards > zsend.max_targets)) { + log_fatal("send", "senders * shards > max targets"); + } + uint64_t num_addrs = blocklist_count_allowed(); + it = iterator_init(zconf.senders, zconf.shard_num, zconf.total_shards, + num_addrs, zconf.ports->port_count); + // determine the source address offset from which we'll send packets + struct in_addr temp; + temp.s_addr = zconf.source_ip_addresses[0]; + log_debug("send", "srcip_first: %s", inet_ntoa(temp)); + temp.s_addr = zconf.source_ip_addresses[zconf.number_source_ips - 1]; + log_debug("send", "srcip_last: %s", inet_ntoa(temp)); + + // process the source port range that ZMap is allowed to use + num_src_ports = zconf.source_port_last - zconf.source_port_first + 1; + log_debug("send", "will send from %u address%s on %hu source ports", + zconf.number_source_ips, + ((zconf.number_source_ips == 1) ? "" : "es"), num_src_ports); + // global initialization for send module + assert(zconf.probe_module); + if (zconf.probe_module->global_initialize) { + if (zconf.probe_module->global_initialize(&zconf)) { + log_fatal( + "send", + "global initialization for probe module failed."); + } + } + // only allow bandwidth or rate + if (zconf.bandwidth > 0 && zconf.rate > 0) { + log_fatal( + "send", + "must specify rate or bandwidth, or neither, not both."); + } + // Convert specified bandwidth to packet rate. This is an estimate using the + // max packet size a probe module will generate. + if (zconf.bandwidth > 0) { + size_t pkt_len = zconf.probe_module->max_packet_length; + pkt_len *= 8; + // 7 byte MAC preamble, 1 byte Start frame, 4 byte CRC, 12 byte + // inter-frame gap + pkt_len += 8 * 24; + // adjust calculated length if less than the minimum size of an + // ethernet frame + if (pkt_len < 84 * 8) { + pkt_len = 84 * 8; + } + // rate is a uint32_t so, don't overflow + if (zconf.bandwidth / pkt_len > 0xFFFFFFFFu) { + zconf.rate = 0; + } else { + zconf.rate = zconf.bandwidth / pkt_len; + if (zconf.rate == 0) { + log_warn( + "send", + "bandwidth %lu bit/s is slower than 1 pkt/s, " + "setting rate to 1 pkt/s", + zconf.bandwidth); + zconf.rate = 1; + } + } + log_debug( + "send", + "using bandwidth %lu bits/s for %zu byte probe, rate set to %d pkt/s", + zconf.bandwidth, pkt_len / 8, zconf.rate); + } + // convert default placeholder to default value + if (zconf.rate == -1) { + // default 10K pps + zconf.rate = 10000; + } + // log rate, if explicitly specified + if (zconf.rate < 0) { + log_fatal("send", "rate impossibly slow"); + } + if (zconf.rate > 0 && zconf.bandwidth <= 0) { + log_debug("send", "rate set to %d pkt/s", zconf.rate); + } + // Get the source hardware address, and give it to the probe + // module + if (!zconf.hw_mac_set) { + if (get_iface_hw_addr(zconf.iface, zconf.hw_mac)) { + log_fatal( + "send", + "ZMap could not retrieve the hardware (MAC) address for " + "the interface \"%s\". You likely do not privileges to open a raw packet socket. " + "Are you running as root or with the CAP_NET_RAW capability? If you are, you " + "may need to manually set the source MAC address with the \"--source-mac\" flag.", + zconf.iface); + return NULL; + } + log_debug( + "send", + "no source MAC provided. " + "automatically detected %02x:%02x:%02x:%02x:%02x:%02x as hw " + "interface for %s", + zconf.hw_mac[0], zconf.hw_mac[1], zconf.hw_mac[2], + zconf.hw_mac[3], zconf.hw_mac[4], zconf.hw_mac[5], + zconf.iface); + } + log_debug("send", "source MAC address %02x:%02x:%02x:%02x:%02x:%02x", + zconf.hw_mac[0], zconf.hw_mac[1], zconf.hw_mac[2], + zconf.hw_mac[3], zconf.hw_mac[4], zconf.hw_mac[5]); + + if (zconf.dryrun) { + log_info("send", "dryrun mode -- won't actually send packets"); + } + // initialize random validation key + validate_init(); + // setup signal handlers for changing scan speed + signal(SIGUSR1, sig_handler_increase_speed); + signal(SIGUSR2, sig_handler_decrease_speed); + zsend.start = now(); + return it; +} + +static inline ipaddr_n_t get_src_ip(ipaddr_n_t dst, int local_offset) +{ + if (zconf.number_source_ips == 1) { + return zconf.source_ip_addresses[0]; + } + return zconf.source_ip_addresses[(ntohl(dst) + local_offset) % + zconf.number_source_ips]; +} + +// one sender thread +int send_run(sock_t st, shard_t *s) +{ + log_debug("send", "send thread started"); + pthread_mutex_lock(&send_mutex); + // Allocate a buffer to hold the outgoing packet + char buf[MAX_PACKET_SIZE]; + // allocate batch + batch_t* batch = create_packet_batch(zconf.batch); + + // OS specific per-thread init + if (send_run_init(st)) { + pthread_mutex_unlock(&send_mutex); + return EXIT_FAILURE; + } + // MAC address length in characters + char mac_buf[(ETHER_ADDR_LEN * 2) + (ETHER_ADDR_LEN - 1) + 1]; + char *p = mac_buf; + for (int i = 0; i < ETHER_ADDR_LEN; i++) { + if (i == ETHER_ADDR_LEN - 1) { + snprintf(p, 3, "%.2x", zconf.hw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", zconf.hw_mac[i]); + p += 3; + } + } + log_debug("send", "source MAC address %s", mac_buf); + void *probe_data; + if (zconf.probe_module->thread_initialize) { + zconf.probe_module->thread_initialize( + buf, zconf.hw_mac, zconf.gw_mac, &probe_data); + } + pthread_mutex_unlock(&send_mutex); + + // adaptive timing to hit target rate + uint64_t count = 0; + uint64_t last_count = count; + double last_time = steady_now(); + uint32_t delay = 0; + int interval = 0; + volatile int vi; + struct timespec ts, rem; + double send_rate = + (double)zconf.rate / + ((double)zconf.senders * zconf.packet_streams); + const double slow_rate = 50; // packets per seconds per thread + // at which it uses the slow methods + long nsec_per_sec = 1000 * 1000 * 1000; + long long sleep_time = nsec_per_sec; + if (zconf.rate > 0) { + delay = 10000; + if (send_rate < slow_rate) { + // set the initial time difference + sleep_time = nsec_per_sec / send_rate; + last_time = steady_now() - (1.0 / send_rate); + } else { + // estimate initial rate + for (vi = delay; vi--;) + ; + delay *= 1 / (steady_now() - last_time) / + ((double)zconf.rate / + (double)zconf.senders); + interval = ((double)zconf.rate / + (double)zconf.senders) / + 20; + last_time = steady_now(); + } + } + // Get the initial IP to scan. + target_t current = shard_get_cur_target(s); + uint32_t current_ip = current.ip; + uint16_t current_port = current.port; + + // If provided a list of IPs to scan, then the first generated address + // might not be on that list. Iterate until the current IP is one the + // list, then start the true scanning process. + if (zconf.list_of_ips_filename) { + while (!pbm_check(zsend.list_of_ips_pbm, current_ip)) { + current = shard_get_next_target(s); + current_ip = current.ip; + current_port = current.port; + if (current.status == ZMAP_SHARD_DONE) { + log_debug( + "send", + "never made it to send loop in send thread %i", + s->thread_id); + goto cleanup; + } + } + } + int attempts = zconf.retries + 1; + uint32_t idx = 0; + while (1) { + // Adaptive timing delay + if (count && delay > 0) { + if (send_rate < slow_rate) { + double t = steady_now(); + double last_rate = (1.0 / (t - last_time)); + + sleep_time *= ((last_rate / send_rate) + 1) / 2; + ts.tv_sec = sleep_time / nsec_per_sec; + ts.tv_nsec = sleep_time % nsec_per_sec; + log_debug("sleep", + "sleep for %d sec, %ld nanoseconds", + ts.tv_sec, ts.tv_nsec); + while (nanosleep(&ts, &rem) == -1) { + } + last_time = t; + } else { + for (vi = delay; vi--;) + ; + if (!interval || (count % interval == 0)) { + double t = steady_now(); + assert(count > last_count); + assert(t > last_time); + double multiplier = + (double)(count - last_count) / + (t - last_time) / + (zconf.rate / zconf.senders); + uint32_t old_delay = delay; + delay *= multiplier; + if (delay == old_delay) { + if (multiplier > 1.0) { + delay *= 2; + } else if (multiplier < 1.0) { + delay *= 0.5; + } + } + last_count = count; + last_time = t; + } + } + } + + // Check if the program has otherwise completed and break out of the send loop. + if (zrecv.complete) { + goto cleanup; + } + if (zconf.max_runtime && + zconf.max_runtime <= now() - zsend.start) { + goto cleanup; + } + + // Check if we've finished this shard or thread before sending each + // packet, regardless of batch size. + if (s->state.max_targets && + s->state.targets_scanned >= s->state.max_targets) { + log_debug( + "send", + "send thread %hhu finished (max targets of %u reached)", + s->thread_id, s->state.max_targets); + goto cleanup; + } + if (s->state.max_packets && + s->state.packets_sent >= s->state.max_packets) { + log_debug( + "send", + "send thread %hhu finished (max packets of %u reached)", + s->thread_id, s->state.max_packets); + goto cleanup; + } + if (current.status == ZMAP_SHARD_DONE) { + log_debug( + "send", + "send thread %hhu finished, shard depleted", + s->thread_id); + goto cleanup; + } + for (int i = 0; i < zconf.packet_streams; i++) { + count++; + uint32_t src_ip = get_src_ip(current_ip, i); + uint32_t validation[VALIDATE_BYTES / + sizeof(uint32_t)]; + validate_gen(src_ip, current_ip, + htons(current_port), + (uint8_t *)validation); + uint8_t ttl = zconf.probe_ttl; + size_t length = 0; + zconf.probe_module->make_packet( + buf, &length, src_ip, current_ip, + htons(current_port), ttl, validation, i, + probe_data); + if (length > MAX_PACKET_SIZE) { + log_fatal( + "send", + "send thread %hhu set length (%zu) larger than MAX (%zu)", + s->thread_id, length, + MAX_PACKET_SIZE); + } + if (zconf.dryrun) { + lock_file(stdout); + zconf.probe_module->print_packet(stdout, + buf); + unlock_file(stdout); + } else { + void *contents = + buf + + zconf.send_ip_pkts * + sizeof(struct ether_header); + length -= (zconf.send_ip_pkts * + sizeof(struct ether_header)); + // add packet to batch and update metadata + // this is an additional memcpy (packet created in buf, buf -> batch) + // but when I modified the TCP SYN module to write packet to batch directly, there wasn't any noticeable speedup. + // Using this approach for readability/minimal changes + memcpy(((uint8_t *)batch->packets) + (batch->len * MAX_PACKET_SIZE), contents, length); + batch->lens[batch->len] = length; + batch->ips[batch->len] = current_ip; + batch->len++; + if (batch->len == batch->capacity) { + // batch is full, sending + int rc = send_batch(st, batch, attempts); + // whether batch succeeds or fails, this was the only attempt. Any re-tries are handled within batch + if (rc < 0) { + log_error("send_batch", "could not send any batch packets: %s", strerror(errno)); + // rc is the last error code if all packets couldn't be sent + s->state.packets_failed += batch->len; + } else { + // rc is number of packets sent successfully, if > 0 + s->state.packets_failed += batch->len - rc; + } + // reset batch length for next batch + batch->len = 0; + idx++; + idx &= 0xFF; + } + } + s->state.packets_sent++; + } + // Track the number of targets (ip,port)s we actually scanned. + s->state.targets_scanned++; + + // Get the next IP to scan + current = shard_get_next_target(s); + current_ip = current.ip; + current_port = current.port; + if (zconf.list_of_ips_filename && + current.status != ZMAP_SHARD_DONE) { + // If we have a list of IPs bitmap, ensure the next IP + // to scan is on the list. + while (!pbm_check(zsend.list_of_ips_pbm, + current_ip)) { + current = shard_get_next_target(s); + current_ip = current.ip; + if (current.status == ZMAP_SHARD_DONE) { + log_debug( + "send", + "send thread %hhu shard finished in get_next_ip_loop depleted", + s->thread_id); + goto cleanup; + } + } + } + } +cleanup: + if (!zconf.dryrun && send_batch(st, batch, attempts) < 0) { + log_error("send_batch cleanup", "could not send remaining batch packets: %s", strerror(errno)); + } + free_packet_batch(batch); + s->cb(s->thread_id, s->arg); + if (zconf.dryrun) { + lock_file(stdout); + fflush(stdout); + unlock_file(stdout); + } + log_debug("send", "thread %hu cleanly finished", s->thread_id); + return EXIT_SUCCESS; +} + +batch_t* create_packet_batch(uint8_t capacity) { + // calculate how many bytes are needed for each component of a batch + int size_of_packet_array = MAX_PACKET_SIZE * capacity; + int size_of_ips_array = sizeof(uint32_t) * capacity; + int size_of_lens_array = sizeof(int) * capacity; + + // allocating batch and associated data structures in single calloc for cache locality + void* batch_and_batch_arrs = calloc(sizeof(batch_t) + size_of_packet_array + size_of_ips_array + size_of_lens_array, sizeof(char)); + // chunk off parts of batch + batch_t* batch = batch_and_batch_arrs; + batch->packets = (char *)batch + sizeof(batch_t); + batch->ips = (uint32_t *)(batch->packets + size_of_packet_array); + batch->lens = (int *) ((char *)batch->ips + size_of_ips_array); + + batch->capacity = capacity; + batch->len = 0; + + return batch; +} + +void free_packet_batch(batch_t* batch) { + // batch was created with a single calloc, so this will free all the component arrays too + free(batch); +} diff --git a/src/send.h b/src/send.h new file mode 100644 index 0000000..58d1d86 --- /dev/null +++ b/src/send.h @@ -0,0 +1,30 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef SEND_H +#define SEND_H + +#include "iterator.h" +#include "socket.h" +#include "./probe_modules/packet.h" + +iterator_t *send_init(void); +int send_run(sock_t, shard_t *); + +typedef struct { + char* packets; + uint32_t* ips; + int* lens; + uint8_t len; + uint8_t capacity; +}batch_t; + +batch_t* create_packet_batch(uint8_t capacity); +void free_packet_batch(batch_t* batch); + +#endif // SEND_H diff --git a/src/shard.c b/src/shard.c new file mode 100644 index 0000000..fe1762f --- /dev/null +++ b/src/shard.c @@ -0,0 +1,193 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include + +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/blocklist.h" +#include "shard.h" +#include "state.h" + +static inline uint16_t extract_port(uint64_t v, uint8_t bits) +{ + uint64_t mask = (1 << bits) - 1; + return (uint16_t)(v & mask); +} + +static inline uint32_t extract_ip(uint64_t v, uint8_t bits) +{ + return (uint32_t)(v >> bits); +} + +static void shard_roll_to_valid(shard_t *s) +{ + uint64_t current_ip_index = (s->current - 1) >> s->bits_for_port; + if (current_ip_index < zsend.max_index) { + return; + } + shard_get_next_target(s); +} + +void shard_init(shard_t *shard, uint16_t shard_idx, uint16_t num_shards, + uint8_t thread_idx, uint8_t num_threads, + uint32_t max_total_targets, uint8_t bits_for_port, + const cycle_t *cycle, shard_complete_cb cb, void *arg) +{ + // Start out by figuring out how many shards we have. A single shard of + // ZMap (set with --shards=N, --shard=n) may have several subshards, if + // ZMap is being ran multithreaded (set with --sender-threads=T). + // + // Total number of subshards is S = N*T. Subshard ID's range from [0, + // N*T). + assert(num_shards > 0); + assert(num_threads > 0); + assert(shard_idx < num_shards); + assert(thread_idx < num_threads); + uint32_t num_subshards = (uint32_t)num_shards * (uint32_t)num_threads; + uint64_t num_elts = cycle->order; + assert(num_subshards < num_elts); + assert(!max_total_targets || (num_subshards <= max_total_targets)); + + // This instance of ZMap will run T subshards, with one subshard per + // thread. This composes a single shard, as specified by the command + // line flag --shard=n. E.g. to run shard with index n, we must run + // subshards with indices the range [n*T, (n+1)*T]. + // + // We can calculate our subshard index i = n*T + t. + uint32_t sub_idx = shard_idx * num_threads + thread_idx; + + // Given i, we want to calculate the start of subshard i. Subshards + // define ranges over exponents of g. They range from [0, Q-1), where Q + // is the number of elements in (the order of) the group generated by + // g. + // + // Let e_b = floor(Q / S) * i + uint64_t exponent_begin = (num_elts / num_subshards) * sub_idx; + + // The stopping exponent is the first element of the next shard. + // + // e_e = floor(Q / S) * ((i + 1) % S) + uint64_t exponent_end = + (num_elts / num_subshards) * ((sub_idx + 1) % num_subshards); + + // We actually offset the begin and end of each cycle. Given an offset + // k, shift each exponent by k modulo Q. + exponent_begin = (exponent_begin + cycle->offset) % num_elts; + exponent_end = (exponent_end + cycle->offset) % num_elts; + + // Multiprecision variants of everything above + mpz_t generator_m, exponent_begin_m, exponent_end_m, prime_m; + mpz_init_set_ui(generator_m, cycle->generator); + mpz_init_set_ui(exponent_begin_m, exponent_begin); + mpz_init_set_ui(exponent_end_m, exponent_end); + mpz_init_set_ui(prime_m, cycle->group->prime); + + // Calculate the first and last points of the shard as powers of g + // modulo p. + mpz_t start_m, stop_m; + mpz_init(start_m); + mpz_init(stop_m); + mpz_powm(start_m, generator_m, exponent_begin_m, prime_m); + mpz_powm(stop_m, generator_m, exponent_end_m, prime_m); + + // Pull the result out as a uint64_t + shard->params.first = (uint64_t)mpz_get_ui(start_m); + shard->params.last = (uint64_t)mpz_get_ui(stop_m); + shard->params.factor = cycle->generator; + shard->params.modulus = cycle->group->prime; + // + shard->bits_for_port = bits_for_port; + + // Set the shard at the beginning. + shard->current = shard->params.first; + + // Set the (thread) id + shard->thread_id = thread_idx; + + // Set max_targets if applicable + if (max_total_targets > 0) { + uint32_t max_targets_this_shard = + max_total_targets / num_subshards; + if (sub_idx < (max_total_targets % num_subshards)) { + ++max_targets_this_shard; + } + shard->state.max_targets = max_targets_this_shard; + } + + // Set the callbacks + shard->cb = cb; + shard->arg = arg; + + // If the beginning of a shard isn't pointing to a valid index in the + // blocklist, find the first element that is. + shard_roll_to_valid(shard); + + // Clear everything + mpz_clear(generator_m); + mpz_clear(exponent_begin_m); + mpz_clear(exponent_end_m); + mpz_clear(prime_m); + mpz_clear(start_m); + mpz_clear(stop_m); +} + +target_t shard_get_cur_target(shard_t *shard) +{ + if (shard->current == ZMAP_SHARD_DONE) { + // shard_roll_to_valid() has rolled to the very end. + return (target_t){ + .ip = 0, .port = 0, .status = ZMAP_SHARD_DONE}; + } + uint32_t ip = extract_ip(shard->current - 1, shard->bits_for_port); + uint16_t port = extract_port(shard->current - 1, shard->bits_for_port); + return (target_t){.ip = (uint32_t)blocklist_lookup_index(ip), + .port = (uint16_t)zconf.ports->ports[port], + .status = ZMAP_SHARD_OK}; +} + +static inline uint64_t shard_get_next_elem(shard_t *shard) +{ + do { + shard->current *= shard->params.factor; + shard->current %= shard->params.modulus; + } while (shard->current >= (1LL << 48)); + return (uint64_t)shard->current; +} + +target_t shard_get_next_target(shard_t *shard) +{ + if (shard->current == ZMAP_SHARD_DONE) { + return (target_t){ + .ip = 0, .port = 0, .status = ZMAP_SHARD_DONE}; + } + while (1) { + uint64_t candidate = shard_get_next_elem(shard); + if (candidate == shard->params.last) { + shard->current = ZMAP_SHARD_DONE; + shard->iterations++; + return (target_t){ + .ip = 0, .port = 0, .status = ZMAP_SHARD_DONE}; + } + uint32_t candidate_ip = + extract_ip(candidate - 1, shard->bits_for_port); + uint16_t candidate_port = + extract_port(candidate - 1, shard->bits_for_port); + if (candidate_ip < zsend.max_index && + candidate_port < zconf.ports->port_count) { + shard->iterations++; + return (target_t){ + .ip = blocklist_lookup_index(candidate_ip), + .port = zconf.ports->ports[candidate_port], + .status = ZMAP_SHARD_OK}; + } + } +} diff --git a/src/shard.h b/src/shard.h new file mode 100644 index 0000000..eaae1dd --- /dev/null +++ b/src/shard.h @@ -0,0 +1,58 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SHARD_H +#define ZMAP_SHARD_H + +#include + +#include "cyclic.h" + +#define ZMAP_SHARD_DONE 0 +#define ZMAP_SHARD_OK 1 + +typedef void (*shard_complete_cb)(uint8_t id, void *arg); + +typedef struct shard { + struct shard_state { + uint64_t packets_sent; + uint32_t targets_scanned; + uint32_t max_targets; + uint32_t max_packets; + uint32_t packets_failed; + uint32_t first_scanned; + } state; + struct shard_params { + uint64_t first; + uint64_t last; + uint64_t factor; + uint64_t modulus; + } params; + uint64_t current; + uint64_t iterations; + uint8_t thread_id; + uint8_t bits_for_port; + shard_complete_cb cb; + void *arg; +} shard_t; + +void shard_init(shard_t *shard, uint16_t shard_idx, uint16_t num_shards, + uint8_t thread_idx, uint8_t num_threads, + uint32_t max_total_targets, uint8_t bits_for_port, + const cycle_t *cycle, shard_complete_cb cb, void *arg); + +typedef struct target { + uint32_t ip; + uint16_t port; + uint8_t status; +} target_t; + +target_t shard_get_cur_target(shard_t *shard); +target_t shard_get_next_target(shard_t *shard); + +#endif /* ZMAP_SHARD_H */ diff --git a/src/socket-bsd.c b/src/socket-bsd.c new file mode 100644 index 0000000..5bb438b --- /dev/null +++ b/src/socket-bsd.c @@ -0,0 +1,88 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" + +#include +#include +#include +#include +#include +#include + +#include "state.h" + +sock_t get_socket(UNUSED uint32_t id) +{ + sock_t sock; + sock.sock = -1; + +#if !(defined(__APPLE__) || defined(__FreeBSD__)) + if (zconf.send_ip_pkts && !zconf.dryrun) { + log_fatal("socket", "iplayer not supported on this flavour of BSD"); + } +#endif + + if (zconf.send_ip_pkts) { + int fd = socket(PF_INET, SOCK_RAW, IPPROTO_RAW); + if (fd == -1) { + log_fatal("socket-bsd", "obtaining socket(PF_INET, SOCK_RAW, IPPROTO_IP) failed: %d %s. You likely do not have privileges to open a raw packet socket. " \ + "Are you running as root?", errno, strerror(errno)); + } + + int yes = 1; + if (setsockopt(fd, IPPROTO_IP, IP_HDRINCL, &yes, sizeof(yes)) == -1) { + // could not inform the kernel that ZMap will be filling out the IP header and the kernel does not need to + log_fatal("socket-bsd", "setsockopt(IP_HDRINCL) failed: %d %s", errno, strerror(errno)); + } + + sock.sock = fd; + } else { + int bpf; + char file[32]; + + // Try to find a valid bpf + for (int i = 0; i < 128; i++) { + snprintf(file, sizeof(file), "/dev/bpf%d", i); + bpf = open(file, O_WRONLY); + if (bpf != -1 || errno != EBUSY) + break; + } + + // Make sure it worked + if (bpf < 0) { + log_fatal("socket-bsd", "could not get an open packet filter"); + } + + // Set up an ifreq to bind to + struct ifreq ifr; + memset(&ifr, 0, sizeof(ifr)); + strlcpy(ifr.ifr_name, zconf.iface, sizeof(ifr.ifr_name)); + + // Bind the bpf to the interface + if (ioctl(bpf, BIOCSETIF, (char *)&ifr) < 0) { + close(bpf); + log_fatal("socket-bsd", "could not bind the packet filter to the interface %s", ifr.ifr_name); + } + + // Inform the interface that the source address will be filled out by ZMap and the kernel doens't need to write it + int write_addr_enable = 1; + if (ioctl(bpf, BIOCSHDRCMPLT, &write_addr_enable) < 0) { + close(bpf); + log_fatal("socket-bsd", "could not set the status of the header complete flag to the packet filter on interface %s", ifr.ifr_name); + } + + sock.sock = bpf; + } + return sock; +} diff --git a/src/socket-linux.c b/src/socket-linux.c new file mode 100644 index 0000000..b1710c7 --- /dev/null +++ b/src/socket-linux.c @@ -0,0 +1,36 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" + +#include "state.h" + +sock_t get_socket(UNUSED uint32_t id) +{ + int sock; + if (zconf.send_ip_pkts) { + sock = socket(AF_PACKET, SOCK_DGRAM, htons(ETH_P_ALL)); + } else { + sock = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); + } + if (sock <= 0) { + log_fatal("send", + "couldn't create socket. " + "Are you root? Error: %s\n", + strerror(errno)); + } + sock_t s; + s.sock = sock; + return s; +} diff --git a/src/socket-pfring.c b/src/socket-pfring.c new file mode 100644 index 0000000..a8f3c7e --- /dev/null +++ b/src/socket-pfring.c @@ -0,0 +1,22 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include "../lib/includes.h" +#include "state.h" + +#include + +sock_t get_socket(uint32_t id) +{ + sock_t sock; + sock.pf.queue = zconf.pf.queues[id]; + sock.pf.buffers = zconf.pf.buffers + 256 * id; + return sock; +} diff --git a/src/socket.c b/src/socket.c new file mode 100644 index 0000000..f6a877e --- /dev/null +++ b/src/socket.c @@ -0,0 +1,31 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" + +sock_t get_dryrun_socket(void) +{ + // we need a socket in order to gather details about the system + // such as source MAC address and IP address. However, because + // we don't want to require root access in order to run dryrun, + // we just create a TCP socket. + int sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock <= 0) { + log_fatal("send", "couldn't create socket. Error: %s\n", + strerror(errno)); + } + sock_t s; + s.sock = sock; + return s; +} diff --git a/src/socket.h b/src/socket.h new file mode 100644 index 0000000..8b9fe84 --- /dev/null +++ b/src/socket.h @@ -0,0 +1,40 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SOCKET_H +#define ZMAP_SOCKET_H + +#include + +#include "../lib/includes.h" + +#ifdef PFRING + +#include + +typedef union { + int sock; + struct { + pfring_zc_queue *queue; + pfring_zc_pkt_buff **buffers; + int idx; + } pf; +} sock_t; + +#else + +typedef struct { + int sock; +} sock_t; + +#endif /* PFRING */ + +sock_t get_dryrun_socket(void); +sock_t get_socket(uint32_t id); + +#endif /* ZMAP_SOCKET_H */ diff --git a/src/state.c b/src/state.c new file mode 100644 index 0000000..cffe3e5 --- /dev/null +++ b/src/state.c @@ -0,0 +1,108 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "state.h" +#include "../lib/logger.h" + +const char *const DEDUP_METHOD_NAMES[] = {"default", "none", "full", "window"}; + +// global configuration and defaults +struct state_conf zconf = { + .allowlist_filename = NULL, + .bandwidth = 0, + .batch = 64, + .blocklist_filename = NULL, + .cooldown_secs = 0, + .custom_metadata_str = NULL, + .data_link_size = 0, + .default_mode = 0, + .dedup_method = 0, + .dedup_window_size = 0, + .dryrun = 0, + .hw_mac = {0}, + .hw_mac_set = 0, + .gw_ip = 0, + .gw_mac = {0}, + .gw_mac_set = 0, + .iface = NULL, + .list_of_ips_count = 0, + .list_of_ips_filename = NULL, + .log_directory = NULL, + .log_file = NULL, + .log_level = LOG_INFO, + .max_results = 0, + .max_runtime = 0, + .max_sendto_failures = -1, + .max_targets = UINT64_MAX, + .metadata_file = NULL, + .metadata_filename = NULL, + .min_hitrate = 0.0, + .no_header_row = 0, + .notes = NULL, + .number_source_ips = 0, + .output_args = NULL, + .output_fields = NULL, + .output_fields_len = 0, + .output_filename = NULL, + .output_filter_str = NULL, + .output_module = NULL, + .packet_streams = 1, + .ports = NULL, + .probe_args = NULL, + .probe_module = NULL, + .probe_ttl = MAXTTL, + .quiet = 0, + .rate = -1, + .raw_output_fields = NULL, + .recv_ready = 0, + .retries = 10, + .seed = 0, + .seed_provided = 0, + .senders = 1, + .send_ip_pkts = 0, + .source_port_first = 32768, // (these are the default + .source_port_last = 61000, // ephemeral range on Linux), + .status_updates_file = NULL, + .syslog = 1}; + +void init_empty_global_configuration(struct state_conf *c) +{ + memset(c->source_ip_addresses, 0, sizeof(c->source_ip_addresses)); +} + +// global sender stats and defaults +struct state_send zsend = { + .start = 0.0, + .finish = 0.0, + .packets_sent = 0, + .targets_scanned = 0, + .warmup = 1, + .complete = 0, + .sendto_failures = 0, + .max_targets = 0, + .list_of_ips_pbm = NULL, +}; + +// global receiver stats and defaults +struct state_recv zrecv = { + .success_unique = 0, + .success_total = 0, + .app_success_unique = 0, + .app_success_total = 0, + .validation_passed = 0, + .validation_failed = 0, + .cooldown_unique = 0, + .cooldown_total = 0, + .failure_total = 0, + .filter_success = 0, + .ip_fragments = 0, + .complete = 0, + .pcap_recv = 0, + .pcap_drop = 0, + .pcap_ifdrop = 0, +}; diff --git a/src/state.h b/src/state.h new file mode 100644 index 0000000..4163138 --- /dev/null +++ b/src/state.h @@ -0,0 +1,216 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef STATE_H +#define STATE_H + +#include +#include + +#include "../lib/includes.h" + +#ifdef PFRING +#include +#endif + +#include "aesrand.h" +#include "fieldset.h" +#include "filter.h" +#include "types.h" + +#define MAX_PACKET_SIZE 4096 +#define MAC_ADDR_LEN_BYTES 6 + +#define DEDUP_METHOD_DEFAULT 0 +#define DEDUP_METHOD_NONE 1 +#define DEDUP_METHOD_FULL 2 +#define DEDUP_METHOD_WINDOW 3 + +extern const char *const DEDUP_METHOD_NAMES[]; + +struct probe_module; +struct output_module; + +struct fieldset_conf { + fielddefset_t defs; + fielddefset_t outdefs; + translation_t translation; + int success_index; + int app_success_index; + int classification_index; +}; + +// global configuration +struct state_conf { + int log_level; + struct port_conf *ports; + port_h_t source_port_first; + port_h_t source_port_last; + // maximum number of packets that the scanner will send before + // terminating + uint64_t max_targets; + // maximum number of seconds that scanner will run before terminating + uint32_t max_runtime; + // maximum number of results before terminating + uint64_t max_results; + // name of network interface that + // will be utilized for sending/receiving + char *iface; + // rate in packets per second + // that the sender will maintain + int rate; + // rate in bits per second + uint64_t bandwidth; + // how many seconds after the termination of the sender will the + // receiver continue to process responses + int cooldown_secs; + // number of sending threads + uint8_t senders; + uint8_t batch; + uint32_t pin_cores_len; + uint32_t *pin_cores; + // should use CLI provided randomization seed instead of generating + // a random seed. + int seed_provided; + uint64_t seed; + aesrand_t *aes; + // generator of the cyclic multiplicative group that is utilized for + // address generation + uint32_t generator; + // sharding options + uint16_t shard_num; + uint16_t total_shards; + int packet_streams; + struct probe_module *probe_module; + char *output_module_name; + struct output_module *output_module; + char *probe_args; + uint8_t probe_ttl; + char *output_args; + macaddr_t gw_mac[MAC_ADDR_LEN_BYTES]; + macaddr_t hw_mac[MAC_ADDR_LEN_BYTES]; + uint32_t gw_ip; + int gw_mac_set; + int hw_mac_set; + in_addr_t source_ip_addresses[256]; + uint32_t number_source_ips; + int send_ip_pkts; + char *output_filename; + char *blocklist_filename; + char *allowlist_filename; + char *list_of_ips_filename; + uint32_t list_of_ips_count; + char *metadata_filename; + FILE *metadata_file; + char *notes; + char *custom_metadata_str; + char **destination_cidrs; + int destination_cidrs_len; + const char *raw_output_fields; + const char **output_fields; + struct output_filter filter; + char *output_filter_str; + struct fieldset_conf fsconf; + int output_fields_len; + char *log_file; + char *log_directory; + char *status_updates_file; + int dryrun; + int quiet; + int ignore_invalid_hosts; + int syslog; + int recv_ready; + int retries; + uint64_t total_allowed; + uint64_t total_disallowed; + int max_sendto_failures; + float min_hitrate; + int data_link_size; + int default_mode; + int no_header_row; + int dedup_method; + int dedup_window_size; +#ifdef PFRING + struct { + pfring_zc_cluster *cluster; + pfring_zc_queue *send; + pfring_zc_queue *recv; + pfring_zc_queue **queues; + pfring_zc_pkt_buff **buffers; + pfring_zc_buffer_pool *prefetches; + } pf; +#endif +}; +extern struct state_conf zconf; + +void init_empty_global_configuration(struct state_conf *c); + +// global sender stats +struct state_send { + double start; + double finish; + uint64_t packets_sent; + uint64_t targets_scanned; + int warmup; + int complete; + uint32_t first_scanned; + uint32_t max_targets; + uint32_t sendto_failures; + uint32_t max_index; + uint16_t max_port_index; + uint8_t **list_of_ips_pbm; +}; +extern struct state_send zsend; + +// global receiver stats +struct state_recv { + // valid responses classified as "success" + uint32_t success_total; + // unique IPs that sent valid responses classified as "success" + uint32_t success_unique; + // valid responses classified as "success" + uint32_t app_success_total; + // unique IPs that sent valid responses classified as "success" + uint32_t app_success_unique; + // valid responses classified as "success" received during cooldown + uint32_t cooldown_total; + // unique IPs that first sent valid "success"es during cooldown + uint32_t cooldown_unique; + // valid responses NOT classified as "success" + uint32_t failure_total; + // valid responses that passed the filter + uint64_t filter_success; + // how many packets did we receive that were marked as being the first + // fragment in a stream + uint32_t ip_fragments; + // metrics about _only_ validate_packet + uint32_t validation_passed; + uint32_t validation_failed; + + int complete; // has the scanner finished sending? + double start; // timestamp of when recv started + double finish; // timestamp of when recv terminated + + // number of packets captured by pcap filter + uint32_t pcap_recv; + // number of packets dropped because there was no room in + // the operating system's buffer when they arrived, because + // packets weren't being read fast enough + uint32_t pcap_drop; + // number of packets dropped by the network interface or its driver. + uint32_t pcap_ifdrop; +}; +extern struct state_recv zrecv; + +struct port_conf { + int port_count; + uint16_t ports[0xFFFF + 1]; + uint8_t *port_bitmap; +}; + +#endif // _STATE_H diff --git a/src/summary.c b/src/summary.c new file mode 100644 index 0000000..2000a5f --- /dev/null +++ b/src/summary.c @@ -0,0 +1,363 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "summary.h" + +#include +#include +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/blocklist.h" + +#include "state.h" +#include "probe_modules/probe_modules.h" +#include "output_modules/output_modules.h" + +#define STRTIME_LEN 1024 + +#include + +void json_metadata(FILE *file) +{ + char send_start_time[STRTIME_LEN + 1]; + assert(dstrftime(send_start_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + zsend.start)); + char send_end_time[STRTIME_LEN + 1]; + assert(dstrftime(send_end_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + zsend.finish)); + char recv_start_time[STRTIME_LEN + 1]; + assert(dstrftime(recv_start_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + zrecv.start)); + char recv_end_time[STRTIME_LEN + 1]; + assert(dstrftime(recv_end_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + zrecv.finish)); + double hitrate = ((double)100 * zrecv.success_unique) / + ((double)zsend.targets_scanned); + + json_object *obj = json_object_new_object(); + + // scanner host name + char hostname[1024]; + if (gethostname(hostname, 1023) < 0) { + log_error("json_metadata", "unable to retrieve local hostname"); + } else { + hostname[1023] = '\0'; + json_object_object_add(obj, "local_hostname", + json_object_new_string(hostname)); + struct hostent *h = gethostbyname(hostname); + if (h) { + json_object_object_add( + obj, "full_hostname", + json_object_new_string(h->h_name)); + } else { + log_error("json_metadata", + "unable to retrieve complete hostname"); + } + } + if (zconf.ports) { + json_object_object_add( + obj, "source_port_first", + json_object_new_int(zconf.source_port_first)); + json_object_object_add( + obj, "source_port_last", + json_object_new_int(zconf.source_port_last)); + + json_object *target_ports = json_object_new_array(); + for (int i = 0; i < zconf.ports->port_count; i++) { + json_object_array_add( + target_ports, + json_object_new_int(zconf.ports->ports[i])); + } + json_object_object_add(obj, "target_ports", target_ports); + } + json_object_object_add(obj, "max_targets", + json_object_new_int(zconf.max_targets)); + json_object_object_add(obj, "max_runtime", + json_object_new_int(zconf.max_runtime)); + json_object_object_add(obj, "max_results", + json_object_new_int(zconf.max_results)); + json_object_object_add(obj, "output_results", + json_object_new_int(zrecv.filter_success)); + if (zconf.iface) { + json_object_object_add(obj, "iface", + json_object_new_string(zconf.iface)); + } + json_object_object_add(obj, "rate", json_object_new_int(zconf.rate)); + json_object_object_add(obj, "bandwidth", + json_object_new_int(zconf.bandwidth)); + json_object_object_add(obj, "cooldown_secs", + json_object_new_int(zconf.cooldown_secs)); + json_object_object_add(obj, "senders", + json_object_new_int(zconf.senders)); + json_object_object_add(obj, "seed", json_object_new_int64(zconf.seed)); + json_object_object_add(obj, "seed_provided", + json_object_new_int64(zconf.seed_provided)); + json_object_object_add(obj, "generator", + json_object_new_int64(zconf.generator)); + json_object_object_add(obj, "hitrate", json_object_new_double(hitrate)); + json_object_object_add(obj, "shard_num", + json_object_new_int(zconf.shard_num)); + json_object_object_add(obj, "total_shards", + json_object_new_int(zconf.total_shards)); + + json_object_object_add(obj, "min_hitrate", + json_object_new_double(zconf.min_hitrate)); + json_object_object_add(obj, "max_sendto_failures", + json_object_new_int(zconf.max_sendto_failures)); + + json_object_object_add(obj, "syslog", + json_object_new_int(zconf.syslog)); + json_object_object_add(obj, "default_mode", + json_object_new_int(zconf.default_mode)); + json_object_object_add(obj, "pcap_recv", + json_object_new_int(zrecv.pcap_recv)); + json_object_object_add(obj, "pcap_drop", + json_object_new_int(zrecv.pcap_drop)); + json_object_object_add(obj, "pcap_ifdrop", + json_object_new_int(zrecv.pcap_ifdrop)); + + json_object_object_add(obj, "ip_fragments", + json_object_new_int(zrecv.ip_fragments)); + json_object_object_add(obj, "blocklist_total_allowed", + json_object_new_int64(zconf.total_allowed)); + json_object_object_add(obj, "blocklist_total_not_allowed", + json_object_new_int64(zconf.total_disallowed)); + json_object_object_add(obj, "validation_passed", + json_object_new_int(zrecv.validation_passed)); + json_object_object_add(obj, "validation_failed", + json_object_new_int(zrecv.validation_failed)); + + // json_object_object_add(obj, "blocklisted", + // json_object_new_int64(zsend.blocklisted)); + // json_object_object_add(obj, "allowlisted", + // json_object_new_int64(zsend.allowlisted)); + json_object_object_add(obj, "first_scanned", + json_object_new_int64(zsend.first_scanned)); + json_object_object_add(obj, "send_to_failures", + json_object_new_int64(zsend.sendto_failures)); + json_object_object_add(obj, "packets_sent", + json_object_new_int64(zsend.packets_sent)); + json_object_object_add(obj, "targets_scanned", + json_object_new_int64(zsend.targets_scanned)); + json_object_object_add(obj, "success_total", + json_object_new_int64(zrecv.success_total)); + json_object_object_add(obj, "success_unique", + json_object_new_int64(zrecv.success_unique)); + if (zconf.fsconf.app_success_index >= 0) { + json_object_object_add( + obj, "app_success_total", + json_object_new_int64(zrecv.app_success_total)); + json_object_object_add( + obj, "app_success_unique", + json_object_new_int64(zrecv.app_success_unique)); + } + json_object_object_add(obj, "success_cooldown_total", + json_object_new_int64(zrecv.cooldown_total)); + json_object_object_add(obj, "success_cooldown_unique", + json_object_new_int64(zrecv.cooldown_unique)); + json_object_object_add(obj, "failure_total", + json_object_new_int64(zrecv.failure_total)); + + json_object_object_add(obj, "packet_streams", + json_object_new_int(zconf.packet_streams)); + json_object_object_add( + obj, "probe_module", + json_object_new_string( + ((probe_module_t *)zconf.probe_module)->name)); + json_object_object_add( + obj, "output_module", + json_object_new_string( + ((output_module_t *)zconf.output_module)->name)); + + json_object_object_add(obj, "send_start_time", + json_object_new_string(send_start_time)); + json_object_object_add(obj, "send_end_time", + json_object_new_string(send_end_time)); + json_object_object_add(obj, "recv_start_time", + json_object_new_string(recv_start_time)); + json_object_object_add(obj, "recv_end_time", + json_object_new_string(recv_end_time)); + + if (zconf.output_filter_str) { + json_object_object_add( + obj, "output_filter", + json_object_new_string(zconf.output_filter_str)); + } + if (zconf.log_file) { + json_object_object_add(obj, "log_file", + json_object_new_string(zconf.log_file)); + } + if (zconf.log_directory) { + json_object_object_add( + obj, "log_directory", + json_object_new_string(zconf.log_directory)); + } + + if (zconf.destination_cidrs_len) { + json_object *cli_dest_cidrs = json_object_new_array(); + for (int i = 0; i < zconf.destination_cidrs_len; i++) { + json_object_array_add( + cli_dest_cidrs, + json_object_new_string(zconf.destination_cidrs[i])); + } + json_object_object_add(obj, "cli_cidr_destinations", + cli_dest_cidrs); + } + if (zconf.probe_args) { + json_object_object_add( + obj, "probe_args", + json_object_new_string(zconf.probe_args)); + } + if (zconf.probe_ttl) { + json_object_object_add(obj, "probe_ttl", + json_object_new_int(zconf.probe_ttl)); + } + if (zconf.output_args) { + json_object_object_add( + obj, "output_args", + json_object_new_string(zconf.output_args)); + } + { + char mac_buf[(MAC_ADDR_LEN * 2) + (MAC_ADDR_LEN - 1) + 1]; + memset(mac_buf, 0, sizeof(mac_buf)); + char *p = mac_buf; + for (int i = 0; i < MAC_ADDR_LEN; i++) { + if (i == MAC_ADDR_LEN - 1) { + snprintf(p, 3, "%.2x", zconf.gw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", zconf.gw_mac[i]); + p += 3; + } + } + json_object_object_add(obj, "gateway_mac", + json_object_new_string(mac_buf)); + } + if (zconf.gw_ip) { + struct in_addr addr; + addr.s_addr = zconf.gw_ip; + json_object_object_add(obj, "gateway_ip", + json_object_new_string(inet_ntoa(addr))); + } + { + char mac_buf[(ETHER_ADDR_LEN * 2) + (ETHER_ADDR_LEN - 1) + 1]; + char *p = mac_buf; + for (int i = 0; i < ETHER_ADDR_LEN; i++) { + if (i == ETHER_ADDR_LEN - 1) { + snprintf(p, 3, "%.2x", zconf.hw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", zconf.hw_mac[i]); + p += 3; + } + } + json_object_object_add(obj, "source_mac", + json_object_new_string(mac_buf)); + } + json_object *source_ips = json_object_new_array(); + for (uint i = 0; i < zconf.number_source_ips; i++) { + struct in_addr temp; + temp.s_addr = zconf.source_ip_addresses[i]; + json_object_array_add(source_ips, json_object_new_string( + strdup(inet_ntoa(temp)))); + } + json_object_object_add(obj, "source_ips", source_ips); + if (zconf.output_filename) { + json_object_object_add( + obj, "output_filename", + json_object_new_string(zconf.output_filename)); + } + if (zconf.blocklist_filename) { + json_object_object_add( + obj, "blocklist_filename", + json_object_new_string(zconf.blocklist_filename)); + } + if (zconf.allowlist_filename) { + json_object_object_add( + obj, "allowlist_filename", + json_object_new_string(zconf.allowlist_filename)); + } + if (zconf.list_of_ips_filename) { + json_object_object_add( + obj, "list_of_ips_filename", + json_object_new_string(zconf.list_of_ips_filename)); + json_object_object_add( + obj, "list_of_ips_count", + json_object_new_int(zconf.list_of_ips_count)); + } + json_object_object_add(obj, "dryrun", + json_object_new_int(zconf.dryrun)); + json_object_object_add(obj, "quiet", json_object_new_int(zconf.quiet)); + json_object_object_add(obj, "log_level", + json_object_new_int(zconf.log_level)); + + json_object_object_add( + obj, "deduplication_method", + json_object_new_string(DEDUP_METHOD_NAMES[zconf.dedup_method])); + if (zconf.dedup_method == DEDUP_METHOD_WINDOW) { + json_object_object_add( + obj, "deduplication_window_size", + json_object_new_int(zconf.dedup_window_size)); + } + + // parse out JSON metadata that was supplied on the command-line + if (zconf.custom_metadata_str) { + json_object *user = + json_tokener_parse(zconf.custom_metadata_str); + if (!user) { + log_error("json-metadata", + "unable to parse user metadata"); + } else { + json_object_object_add(obj, "user-metadata", user); + } + } + + if (zconf.notes) { + json_object_object_add(obj, "notes", + json_object_new_string(zconf.notes)); + } + + // add blocklisted and allowlisted CIDR blocks + bl_cidr_node_t *b = get_blocklisted_cidrs(); + if (b) { + json_object *blocklisted_cidrs = json_object_new_array(); + do { + char cidr[50]; + struct in_addr addr; + addr.s_addr = b->ip_address; + sprintf(cidr, "%s/%i", inet_ntoa(addr), b->prefix_len); + json_object_array_add(blocklisted_cidrs, + json_object_new_string(cidr)); + } while (b && (b = b->next)); + json_object_object_add(obj, "blocklisted_networks", + blocklisted_cidrs); + } + + b = get_allowlisted_cidrs(); + if (b) { + json_object *allowlisted_cidrs = json_object_new_array(); + do { + char cidr[50]; + struct in_addr addr; + addr.s_addr = b->ip_address; + sprintf(cidr, "%s/%i", inet_ntoa(addr), b->prefix_len); + json_object_array_add(allowlisted_cidrs, + json_object_new_string(cidr)); + } while (b && (b = b->next)); + json_object_object_add(obj, "allowlisted_networks", + allowlisted_cidrs); + } + + fprintf(file, "%s\n", json_object_to_json_string(obj)); + json_object_put(obj); +} diff --git a/src/summary.h b/src/summary.h new file mode 100644 index 0000000..8d1c8a9 --- /dev/null +++ b/src/summary.h @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_SUMMARY_H +#define ZMAP_SUMMARY_H + +#include + +void json_metadata(FILE *); + +#endif /* ZMAP_SUMMARY_H */ diff --git a/src/tests/test_harness.c b/src/tests/test_harness.c new file mode 100644 index 0000000..ad4ab0d --- /dev/null +++ b/src/tests/test_harness.c @@ -0,0 +1,97 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#include "aesrand.h" +#include "send.h" +#include "recv.h" +#include "state.h" +#include "monitor.h" +#include "get_gateway.h" +#include "filter.h" +#include "summary.h" + +#include "output_modules/output_modules.h" +#include "probe_modules/probe_modules.h" +#include "output_modules/module_json.h" +#include "ztopt.h" + +int test_recursive_fieldsets(void) +{ + fieldset_t *outer = fs_new_fieldset(NULL); + fieldset_t *inner = fs_new_fieldset(NULL); + + fieldset_t *repeated = fs_new_repeated_string(0); + assert(repeated->type == FS_REPEATED); + assert(repeated->len == 0); + assert(repeated->inner_type == FS_STRING); + for (int i = 0; i < 10; i++) { + fs_add_string(repeated, NULL, (char *)"hello world!", 0); + } + fs_add_repeated(outer, (char *)"repeatedstuff", repeated); + fs_add_string(outer, "name", strdup("value"), 0); + fs_add_string(inner, "name2", strdup("value2"), 0); + fs_add_fieldset(outer, "inner", inner); + + print_json_fieldset(outer); + fs_free(outer); + + return EXIT_SUCCESS; +} + +int main(UNUSED int argc, UNUSED char **argv) +{ + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + assert(params); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + // Handle help text and version + if (args.help_given) { + cmdline_parser_print_help(); + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + for (int i = 0; i < 100000000; i++) + test_recursive_fieldsets(); + return EXIT_SUCCESS; +} diff --git a/src/topt.ggo.in b/src/topt.ggo.in new file mode 100644 index 0000000..bd02dfa --- /dev/null +++ b/src/topt.ggo.in @@ -0,0 +1,38 @@ +# ZTee Copyright 2014 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# ztee option description to be processed by gengetopt + +package "ztee" +version "@ZMAP_VERSION@" +purpose "A buffering output splitter" + +section "Basic arguments" + +option "success-only" - "Only write to stdout rows where success=1 or success=true" + optional +option "monitor" m "Print monitor data to stderr" + optional +option "status-updates-file" u "File to write status updates, in CSV format" + typestr="monitor.csv" + optional string +option "log-file" l "File to log errors, etc. to" + optional string +option "raw" r "Ignore input formatting and pass through raw input" + optional + +section "Additional options" + +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +text "\nExamples:\n\ + zmap -p 80 -o - | ztee zmap.csv (save zmap output to zmap.csv and output all IP addresses to stdout)\n\ + zmap -p 80 --output-fields=* -o - | ztee --success-only zmap.csv (save all zmap output to zmap.csv, print IPs from successful rows to stdout)\n\ + zmap -p 80 -o - | ztee -u status.csv zmap.csv (save zmap output to zmap.csv, write status updates to status.csv, print all IPs to stdout)\n\ + echo \"hello, ztee\" | ztee --raw out.txt (write text to out.txt and to stdout, like tee)" diff --git a/src/topt_compat.c b/src/topt_compat.c new file mode 100644 index 0000000..167ecd5 --- /dev/null +++ b/src/topt_compat.c @@ -0,0 +1,25 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "topt.c" diff --git a/src/utility.c b/src/utility.c new file mode 100644 index 0000000..ecd320f --- /dev/null +++ b/src/utility.c @@ -0,0 +1,71 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "utility.h" + +#include +#include + +#include "state.h" +#include "../lib/logger.h" + +in_addr_t string_to_ip_address(char *t) +{ + in_addr_t r = inet_addr(t); + if (r == INADDR_NONE) { + log_fatal("send", "invalid ip address: `%s'", t); + } + return r; +} + +void add_to_array(char *to_add) +{ + if (zconf.number_source_ips >= 256) { + // log fatal here + log_fatal("parse", "over 256 source IP addresses provided"); + } + log_debug("SEND", "ipaddress: %s\n", to_add); + zconf.source_ip_addresses[zconf.number_source_ips] = + string_to_ip_address(to_add); + zconf.number_source_ips++; +} + +void parse_source_ip_addresses(char given_string[]) +{ + char *dash = strchr(given_string, '-'); + char *comma = strchr(given_string, ','); + if (dash && comma) { + *comma = '\0'; + parse_source_ip_addresses(given_string); + parse_source_ip_addresses(comma + 1); + } else if (comma) { + while (comma) { + *comma = '\0'; + add_to_array(given_string); + given_string = comma + 1; + comma = strchr(given_string, ','); + if (!comma) { + add_to_array(given_string); + } + } + } else if (dash) { + *dash = '\0'; + log_debug("SEND", "address: %s\n", given_string); + log_debug("SEND", "address: %s\n", dash + 1); + in_addr_t start = ntohl(string_to_ip_address(given_string)); + in_addr_t end = ntohl(string_to_ip_address(dash + 1)) + 1; + while (start != end) { + struct in_addr temp; + temp.s_addr = htonl(start); + add_to_array(strdup(inet_ntoa(temp))); + start++; + } + } else { + add_to_array(given_string); + } +} diff --git a/src/utility.h b/src/utility.h new file mode 100644 index 0000000..dbf0258 --- /dev/null +++ b/src/utility.h @@ -0,0 +1,22 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UTILITY_H +#define UTILITY_H + +void parse_source_ip_addresses(char given_string[]); + +#endif // UTILITY_H diff --git a/src/validate.c b/src/validate.c new file mode 100644 index 0000000..d41738f --- /dev/null +++ b/src/validate.c @@ -0,0 +1,54 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include "../lib/rijndael-alg-fst.h" +#include "../lib/random.h" +#include "../lib/logger.h" +#include "validate.h" + +#define AES_ROUNDS 10 +#define AES_BLOCK_WORDS 4 +#define AES_KEY_BYTES 16 + +static int inited = 0; +static uint32_t aes_sched[(AES_ROUNDS + 1) * 4]; + +void validate_init(void) +{ + uint8_t key[AES_KEY_BYTES]; + if (!random_bytes(key, AES_KEY_BYTES)) { + log_fatal("validate", "couldn't get random bytes"); + } + if (rijndaelKeySetupEnc(aes_sched, key, AES_KEY_BYTES * 8) != + AES_ROUNDS) { + log_fatal("validate", "couldn't initialize AES key"); + } + inited = 1; +} + +void validate_gen(const uint32_t src, const uint32_t dst, + const uint16_t dst_port, uint8_t output[VALIDATE_BYTES]) +{ + validate_gen_ex(src, dst, (uint32_t)dst_port, 0, output); +} + +void validate_gen_ex(const uint32_t input0, const uint32_t input1, + const uint32_t input2, const uint32_t input3, + uint8_t output[VALIDATE_BYTES]) +{ + assert(inited); + + uint32_t aes_input[AES_BLOCK_WORDS]; + aes_input[0] = input0; + aes_input[1] = input1; + aes_input[2] = input2; + aes_input[3] = input3; + rijndaelEncrypt(aes_sched, AES_ROUNDS, (uint8_t *)aes_input, output); +} diff --git a/src/validate.h b/src/validate.h new file mode 100644 index 0000000..d910d76 --- /dev/null +++ b/src/validate.h @@ -0,0 +1,21 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef VALIDATE_H +#define VALIDATE_H + +#define VALIDATE_BYTES 16 + +void validate_init(void); +void validate_gen(const uint32_t src, const uint32_t dst, + const uint16_t dst_port, uint8_t output[VALIDATE_BYTES]); +void validate_gen_ex(const uint32_t input0, const uint32_t input1, + const uint32_t input2, const uint32_t input3, + uint8_t output[VALIDATE_BYTES]); + +#endif //_VALIDATE_H diff --git a/src/zblocklist.1 b/src/zblocklist.1 new file mode 100644 index 0000000..eef8ae7 --- /dev/null +++ b/src/zblocklist.1 @@ -0,0 +1,60 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "ZBLOCKLIST" "1" "August 2021" "ZMap" "zblocklist" +. +.SH "NAME" +\fBzblocklist\fR \- zmap IP blocklist tool +. +.SH "SYNOPSIS" +zblocklist [ \-b ] [ \-w ] [ OPTIONS\.\.\. ] +. +.SH "DESCRIPTION" +\fIZBlacklist\fR is a network tool for limiting and deduplicating a list of IP addresses using a blocklist or allowlist\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fB\-b\fR, \fB\-\-blocklist\-file=path\fR +File of subnets to exclude, in CIDR notation, one\-per line\. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special\-purpose addresses\. An example blocklist file \fBblocklist\.conf\fR for this purpose\. +. +.TP +\fB\-w\fR, \fB\-\-allowlist\-file=name\fR +File of subnets to include, in CIDR notation, one\-per line\. All other subnets will be excluded\. +. +.TP +\fB\-l\fR, \fB\-\-log\-file=name\fR +File to log to\. +. +.TP +\fB\-\-disable\-syslog\fR +Disable logging messages to syslog\. +. +.TP +\fB\-v\fR, \fB\-\-verbosity\fR +Level of log detail (0\-5, default=3) +. +.TP +\fB\-\-no\-duplicate\-checking\fR +Don\'t deduplicate input addresses\. Default is false\. +. +.TP +\fB\-\-ignore\-blocklist\-errors\fR +Ignore invalid, malformed, or unresolvable entries in the blocklist/allowlist\. Default is false\. +. +.TP +\fB\-\-ignore\-input\-errors\fR +Don\'t print invalid entries in the input\. Default is false\. +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-h\fR, \fB\-\-help\fR +Print help and exit +. +.TP +\fB\-V\fR, \fB\-\-version\fR +Print version and exit + diff --git a/src/zblocklist.1.html b/src/zblocklist.1.html new file mode 100644 index 0000000..2911b31 --- /dev/null +++ b/src/zblocklist.1.html @@ -0,0 +1,120 @@ + + + + + + zblocklist(1) - zmap IP blocklist tool + + + + +
+ + + +
    +
  1. zblocklist(1)
    2. +
  2. zblocklist
    3. +
  3. zblocklist(1)
    4. +
+ +

NAME

+

+ zblocklist - zmap IP blocklist tool +

+ +

SYNOPSIS

+ +

zblocklist [ -b <blocklist> ] [ -w <allowlist> ] [ OPTIONS... ]

+ +

DESCRIPTION

+ +

ZBlacklist is a network tool for limiting and deduplicating a list of +IP addresses using a blocklist or allowlist.

+ +

OPTIONS

+ +

BASIC OPTIONS

+ +
+
-b, --blocklist-file=path

File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file blocklist.conf for this purpose.

+
-w, --allowlist-file=name

File of subnets to include, in CIDR notation, one-per line. All other +subnets will be excluded.

+
-l, --log-file=name

File to log to.

+
--disable-syslog

Disable logging messages to syslog.

+
-v, --verbosity

Level of log detail (0-5, default=3)

+
--no-duplicate-checking

Don't deduplicate input addresses. Default is false.

+
--ignore-blocklist-errors

Ignore invalid, malformed, or unresolvable entries in the +blocklist/allowlist. Default is false.

+
--ignore-input-errors

Don't print invalid entries in the input. Default is false.

+
+ + +

ADDITIONAL OPTIONS

+ +
+
-h, --help

Print help and exit

+
-V, --version

Print version and exit

+
+ + + +
    +
  1. ZMap
    2. +
  2. August 2021
    3. +
  3. zblocklist(1)
    4. +
+ +
+ + diff --git a/src/zblocklist.1.ronn b/src/zblocklist.1.ronn new file mode 100644 index 0000000..c2e8fb5 --- /dev/null +++ b/src/zblocklist.1.ronn @@ -0,0 +1,53 @@ +zblocklist(1) - zmap IP blocklist tool +====================================== + +## SYNOPSIS + +zblocklist [ -b <blocklist> ] [ -w <allowlist> ] [ OPTIONS... ] + +## DESCRIPTION + +*ZBlacklist* is a network tool for limiting and deduplicating a list of +IP addresses using a blocklist or allowlist. + +## OPTIONS + +### BASIC OPTIONS ### + + * `-b`, `--blocklist-file=path`: + File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file **blocklist.conf** for this purpose. + + * `-w`, `--allowlist-file=name`: + File of subnets to include, in CIDR notation, one-per line. All other + subnets will be excluded. + + * `-l`, `--log-file=name`: + File to log to. + + * `--disable-syslog`: + Disable logging messages to syslog. + + * `-v`, `--verbosity`: + Level of log detail (0-5, default=3) + + * `--no-duplicate-checking`: + Don't deduplicate input addresses. Default is false. + + * `--ignore-blocklist-errors`: + Ignore invalid, malformed, or unresolvable entries in the + blocklist/allowlist. Default is false. + + * `--ignore-input-errors`: + Don't print invalid entries in the input. Default is false. + + +### ADDITIONAL OPTIONS ### + + * `-h`, `--help`: + Print help and exit + + * `-V`, `--version`: + Print version and exit diff --git a/src/zblocklist.c b/src/zblocklist.c new file mode 100644 index 0000000..ec9da18 --- /dev/null +++ b/src/zblocklist.c @@ -0,0 +1,263 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * ZBlocklist is a simple utility that (1) excludes IP addresses on a specified + * blocklist from being scanned, and (2) ensures the uniqueness of output + * addresses such that no host is scanned twice. ZBlocklist takes in a list + * of addresses on stdin and outputs addresses that are acceptable to scan + * on stdout. The utility uses the blocklist data structures from ZMap for + * checking scan eligibility and a paged bitmap for duplicate prevention. + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/pbm.h" + +#include "zbopt.h" + +// struct zbl_stats { +// uint32_t cidr_entries; +// uint32_t allowed_addrs; +// uint32_t input_addrs; +// uint32_t uniq_input_addrs; +// uint32_t blocked_addrs; +// uint32_t output_addrs; +// uint32_t duplicates; +//}; + +#undef MIN +#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) + +// allow 1mb lines + newline + \0 +#define MAX_LINE_LENGTH 1024 * 1024 + 2 + +static inline char *zmin(char *a, char *b) +{ + if (a && !b) + return a; + else if (b && !a) + return b; + else + return MIN(a, b); +} + +struct zbl_conf { + char *blocklist_filename; + char *allowlist_filename; + char *log_filename; + int check_duplicates; + int ignore_blocklist_errors; + int ignore_input_errors; + int verbosity; + int disable_syslog; + // struct zbl_stats stats; +}; + +#define SET_IF_GIVEN(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = args.ARG##_arg; \ + }; \ + } +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char **argv) +{ + struct zbl_conf conf; + conf.verbosity = 3; + memset(&conf, 0, sizeof(struct zbl_conf)); + int no_dupchk_pres = 0; + conf.ignore_blocklist_errors = 0; + conf.ignore_input_errors = 0; + + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + assert(params); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + // Handle help text and version + if (args.help_given) { + cmdline_parser_print_help(); + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + // Set the log file and metadata file + if (args.log_file_given) { + conf.log_filename = strdup(args.log_file_arg); + } + if (args.verbosity_given) { + conf.verbosity = args.verbosity_arg; + } + + // Blocklist and allowlist + if (args.blocklist_file_given) { + conf.blocklist_filename = strdup(args.blocklist_file_arg); + } + if (args.allowlist_file_given) { + conf.allowlist_filename = strdup(args.allowlist_file_arg); + } + + // Read the boolean flags + SET_BOOL(no_dupchk_pres, no_duplicate_checking); + conf.check_duplicates = !no_dupchk_pres; + SET_BOOL(conf.ignore_blocklist_errors, ignore_blocklist_errors); + SET_BOOL(conf.ignore_input_errors, ignore_input_errors); + SET_BOOL(conf.disable_syslog, disable_syslog); + + // initialize logging + FILE *logfile = stderr; + if (conf.log_filename) { + logfile = fopen(conf.log_filename, "w"); + if (!logfile) { + fprintf( + stderr, + "FATAL: unable to open specified logfile (%s)\n", + conf.log_filename); + exit(1); + } + } + if (log_init(logfile, conf.verbosity, !conf.disable_syslog, + "zblocklist")) { + fprintf(stderr, "FATAL: unable able to initialize logging\n"); + exit(1); + } + + if (!conf.blocklist_filename && !conf.allowlist_filename) { + log_fatal("zblocklist", + "must specify either a allowlist or blocklist file"); + } + + // parse blocklist + if (conf.blocklist_filename) { + log_debug("zblocklist", "blocklist file at %s to be used", + conf.blocklist_filename); + } else { + log_debug("zblocklist", "no blocklist file specified"); + } + if (conf.blocklist_filename && + access(conf.blocklist_filename, R_OK) == -1) { + log_fatal("zblocklist", + "unable to read specified blocklist file (%s)", + conf.blocklist_filename); + } + if (conf.allowlist_filename) { + log_debug("zblocklist", "allowlist file at %s to be used", + conf.allowlist_filename); + } else { + log_debug("zblocklist", "no allowlist file specified"); + } + if (conf.allowlist_filename && + access(conf.allowlist_filename, R_OK) == -1) { + log_fatal("zblocklist", + "unable to read specified allowlist file (%s)", + conf.allowlist_filename); + } + + if (blocklist_init(conf.allowlist_filename, conf.blocklist_filename, + NULL, 0, NULL, 0, conf.ignore_blocklist_errors)) { + log_fatal("zmap", "unable to initialize blocklist / allowlist"); + } + // initialize paged bitmap + uint8_t **seen = NULL; + if (conf.check_duplicates) { + seen = pbm_init(); + if (!seen) { + log_fatal("zblocklist", + "unable to initialize paged bitmap"); + } + } + // process addresses + char *line = malloc(MAX_LINE_LENGTH); + assert(line); + char *original = malloc(MAX_LINE_LENGTH); + assert(original); + while (fgets(line, MAX_LINE_LENGTH, stdin) != NULL) { + size_t len = strlen(line); + if (len >= (MAX_LINE_LENGTH - 1)) { + log_fatal("zblocklist", + "received line longer than max length: %i", + MAX_LINE_LENGTH); + } + // remove new line + memcpy(original, line, len + 1); + char *n = + zmin(zmin(zmin(zmin(strchr(line, '\n'), strchr(line, ',')), + strchr(line, '\t')), + strchr(line, ' ')), + strchr(line, '#')); + assert(n); + n[0] = 0; + log_debug("zblocklist", "input value %s", line); + // parse into int + struct in_addr addr; + if (!inet_aton(line, &addr)) { + log_warn("zblocklist", "invalid input address: %s", + line); + if (!conf.ignore_input_errors) { + printf("%s", original); + } + continue; + } + if (conf.check_duplicates) { + if (pbm_check(seen, ntohl(addr.s_addr))) { + log_debug("zblocklist", + "%s is a duplicate: skipped", line); + continue; + } else { + log_debug("zblocklist", + "%s not a duplicate: skipped", line); + } + } else { + log_debug("zblocklist", "no duplicate checking for %s", + line); + } + // check if in blocklist + if (blocklist_is_allowed(addr.s_addr)) { + if (conf.check_duplicates) { + if (!pbm_check(seen, ntohl(addr.s_addr))) { + pbm_set(seen, ntohl(addr.s_addr)); + printf("%s", original); + } + } else { + printf("%s", original); + } + } + } + return EXIT_SUCCESS; +} diff --git a/src/zbopt.ggo.in b/src/zbopt.ggo.in new file mode 100644 index 0000000..46359f2 --- /dev/null +++ b/src/zbopt.ggo.in @@ -0,0 +1,43 @@ +# ZBlacklist Copyright 2014 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# zblocklist option description to be processed by gengetopt + +package "zblocklist" +version "@ZMAP_VERSION@" +purpose "A tool for limiting and deduplicating a list of IP addresses" + +section "Basic arguments" + +option "blocklist-file" b "File of subnets to exclude, in CIDR notation, one-per line." + optional string +option "allowlist-file" w "File of subnets to include, in CIDR notation, one-per line." + optional string +option "log-file" l "File to log to" + optional string +option "verbosity" v "Set log level verbosity (0-5, default 3)" + default="3" + optional int +option "no-duplicate-checking" - "Don't deduplicate IP addresses (default false)" + optional +option "ignore-blocklist-errors" - "Ignore invalid entries in the blocklist/allowlist (default false)" + optional +option "ignore-input-errors" - "Don't print invalid entries in the input (default false)" + optional +option "disable-syslog" - "Disables logging messages to syslog" + optional + +section "Additional options" + +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +section "Notes" + +text + "At least one of --allowlist-file or --blocklist-file must be specified. Blacklist files take precedence over allowlist files when both are specified. This results in an output of {allowlist - blocklist}." diff --git a/src/zbopt_compat.c b/src/zbopt_compat.c new file mode 100644 index 0000000..5928c1a --- /dev/null +++ b/src/zbopt_compat.c @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "zbopt.c" diff --git a/src/ziterate.1 b/src/ziterate.1 new file mode 100644 index 0000000..60401a4 --- /dev/null +++ b/src/ziterate.1 @@ -0,0 +1,74 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "ZITERATE" "1" "September 2023" "ZMap" "ziterate" +. +.SH "NAME" +\fBziterate\fR \- ZMap IP permutation generation file +. +.SH "SYNOPSIS" +ziterate [ \-b ] [ \-w ] [ OPTIONS\.\.\. ] +. +.SH "DESCRIPTION" +\fIZIterate\fR is a network tool that will produce IPv4 addresses in a psuedorandom order similar to how ZMap generates random addresses to be scanned\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fB\-p\fR, \fB\-\-target\-ports=port(s)\fR +List of TCP/UDP ports and/or port ranges to scan\. (e\.g\., 80,443,100\-105)\. Use \'*\' to scan all ports, including port 0\. If no port is specified, ziterate will output only IPs\. +. +.TP +\fB\-b\fR, \fB\-\-blocklist\-file=path\fR +File of subnets to exclude, in CIDR notation, one\-per line\. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special\-purpose addresses\. An example blocklist file \fBblocklist\.conf\fR for this purpose\. +. +.TP +\fB\-w\fR, \fB\-\-allowlist\-file=name\fR +File of subnets to include, in CIDR notation, one\-per line\. All other subnets will be excluded\. +. +.TP +\fB\-l\fR, \fB\-\-log\-file=name\fR +File to log to\. +. +.TP +\fB\-\-disable\-syslog\fR +Disable logging messages to syslog\. +. +.TP +\fB\-v\fR, \fB\-\-verbosity\fR +Level of log detail (0\-5, default=3) +. +.TP +\fB\-\-ignore\-blocklist\-errors\fR +Ignore invalid entries in the blocklist\. Default is false\. +. +.TP +\fB\-\-seed=n\fR +Seed used to select address permutation\. +. +.TP +\fB\-n\fR, \fB\-\-max\-targets=n\fR +Cap number of IPs to generate (as a number or a percentage of the address space) +. +.SS "SHARDING" +. +.TP +\fB\-\-shards=n\fR +Total number of shards\. +. +.TP +\fB\-\-shard=n\fR +Shard this scan is targeting\. Zero indexed\. +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-h\fR, \fB\-\-help\fR +Print help text and exit\. +. +.TP +\fB\-V\fR, \fB\-\-version\fR +Print version and exit\. + diff --git a/src/ziterate.1.html b/src/ziterate.1.html new file mode 100644 index 0000000..4c8c442 --- /dev/null +++ b/src/ziterate.1.html @@ -0,0 +1,130 @@ + + + + + + ziterate(1) - ZMap IP permutation generation file + + + + +
+ + + +
    +
  1. ziterate(1)
    2. +
  2. ziterate
    3. +
  3. ziterate(1)
    4. +
+ +

NAME

+

+ ziterate - ZMap IP permutation generation file +

+ +

SYNOPSIS

+ +

ziterate [ -b <blocklist> ] [ -w <allowlist> ] [ OPTIONS... ]

+ +

DESCRIPTION

+ +

ZIterate is a network tool that will produce IPv4 addresses in a psuedorandom +order similar to how ZMap generates random addresses to be scanned.

+ +

OPTIONS

+ +

BASIC OPTIONS

+ +
+
-p, --target-ports=port(s)

List of TCP/UDP ports and/or port ranges to scan. (e.g., 80,443,100-105). + Use '*' to scan all ports, including port 0. If no port is specified, + ziterate will output only IPs.

+
-b, --blocklist-file=path

File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file blocklist.conf for this purpose.

+
-w, --allowlist-file=name

File of subnets to include, in CIDR notation, one-per line. All other +subnets will be excluded.

+
-l, --log-file=name

File to log to.

+
--disable-syslog

Disable logging messages to syslog.

+
-v, --verbosity

Level of log detail (0-5, default=3)

+
--ignore-blocklist-errors

Ignore invalid entries in the blocklist. Default is false.

+
--seed=n

Seed used to select address permutation.

+
-n, --max-targets=n

Cap number of IPs to generate (as a number or a percentage of the address space)

+
+ + +

SHARDING

+ +
+
--shards=n

Total number of shards.

+
--shard=n

Shard this scan is targeting. Zero indexed.

+
+ + +

ADDITIONAL OPTIONS

+ +
+
-h, --help

Print help text and exit.

+
-V, --version

Print version and exit.

+
+ + + +
    +
  1. ZMap
    2. +
  2. September 2023
    3. +
  3. ziterate(1)
    4. +
+ +
+ + diff --git a/src/ziterate.1.ronn b/src/ziterate.1.ronn new file mode 100644 index 0000000..9d11311 --- /dev/null +++ b/src/ziterate.1.ronn @@ -0,0 +1,66 @@ +ziterate(1) - ZMap IP permutation generation file +================================================= + +## SYNOPSIS + +ziterate [ -b <blocklist> ] [ -w <allowlist> ] [ OPTIONS... ] + +## DESCRIPTION + +*ZIterate* is a network tool that will produce IPv4 addresses in a psuedorandom +order similar to how ZMap generates random addresses to be scanned. + +## OPTIONS + +### BASIC OPTIONS ### + + * `-p`, `--target-ports=port(s)`: + List of TCP/UDP ports and/or port ranges to scan. (e.g., 80,443,100-105). + Use '*' to scan all ports, including port 0. If no port is specified, + ziterate will output only IPs. + + * `-b`, `--blocklist-file=path`: + File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file **blocklist.conf** for this purpose. + + * `-w`, `--allowlist-file=name`: + File of subnets to include, in CIDR notation, one-per line. All other + subnets will be excluded. + + * `-l`, `--log-file=name`: + File to log to. + + * `--disable-syslog`: + Disable logging messages to syslog. + + * `-v`, `--verbosity`: + Level of log detail (0-5, default=3) + + * `--ignore-blocklist-errors`: + Ignore invalid entries in the blocklist. Default is false. + + * `--seed=n`: + Seed used to select address permutation. + + * `-n`, `--max-targets=n`: + Cap number of IPs to generate (as a number or a percentage of the address space) + + +### SHARDING ### + + * `--shards=n`: + Total number of shards. + + * `--shard=n`: + Shard this scan is targeting. Zero indexed. + + +### ADDITIONAL OPTIONS ### + + * `-h`, `--help`: + Print help text and exit. + + * `-V`, `--version`: + Print version and exit. diff --git a/src/ziterate.c b/src/ziterate.c new file mode 100644 index 0000000..5f5c2cb --- /dev/null +++ b/src/ziterate.c @@ -0,0 +1,246 @@ +/* + * ZMap Copyright 2023 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * ZIterate is a simple utility that will iteratate over the IPv4 + * space in a pseudo-random fashion, utilizing the sharding capabilities * of + * ZMap. + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#include "iterator.h" +#include "ports.h" +#include "state.h" +#include "validate.h" +#include "zitopt.h" + +struct zit_conf { + char *blocklist_filename; + char *allowlist_filename; + char **destination_cidrs; + int destination_cidrs_len; + char *log_filename; + int check_duplicates; + int ignore_errors; + int verbosity; + int disable_syslog; + + // sharding options + uint16_t shard_num; + uint16_t total_shards; + uint64_t seed; + aesrand_t *aes; + uint32_t max_hosts; +}; + +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char **argv) +{ + struct zit_conf conf; + + memset(&conf, 0, sizeof(struct zit_conf)); + conf.verbosity = 3; + conf.ignore_errors = 0; + + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + assert(params); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + // Handle help text and version + if (args.help_given) { + cmdline_parser_print_help(); + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + // Set the log file and metadata file + if (args.log_file_given) { + conf.log_filename = strdup(args.log_file_arg); + } + if (args.verbosity_given) { + conf.verbosity = args.verbosity_arg; + } + // Read the boolean flags + SET_BOOL(conf.ignore_errors, ignore_blocklist_errors); + SET_BOOL(conf.disable_syslog, disable_syslog); + + // initialize logging + FILE *logfile = stderr; + if (conf.log_filename) { + logfile = fopen(conf.log_filename, "w"); + if (!logfile) { + fprintf( + stderr, + "FATAL: unable to open specified logfile (%s)\n", + conf.log_filename); + exit(1); + } + } + if (log_init(logfile, conf.verbosity, !conf.disable_syslog, + "ziterate")) { + fprintf(stderr, "FATAL: unable able to initialize logging\n"); + exit(1); + } + + // Blocklist and allowlist + if (args.blocklist_file_given) { + conf.blocklist_filename = strdup(args.blocklist_file_arg); + } + if (args.allowlist_file_given) { + conf.allowlist_filename = strdup(args.allowlist_file_arg); + } + conf.destination_cidrs = args.inputs; + conf.destination_cidrs_len = args.inputs_num; + // max targets + if (args.max_targets_given) { + conf.max_hosts = parse_max_hosts(args.max_targets_arg); + } + + // sanity check blocklist file + if (conf.blocklist_filename) { + log_debug("ziterate", "blocklist file at %s to be used", + conf.blocklist_filename); + } else { + log_debug("ziterate", "no blocklist file specified"); + } + if (conf.blocklist_filename && + access(conf.blocklist_filename, R_OK) == -1) { + log_fatal("ziterate", + "unable to read specified blocklist file (%s)", + conf.blocklist_filename); + } + + // sanity check allowlist file + if (conf.allowlist_filename) { + log_debug("ziterate", "allowlist file at %s to be used", + conf.allowlist_filename); + } else { + log_debug("ziterate", "no allowlist file specified"); + } + if (conf.allowlist_filename && + access(conf.allowlist_filename, R_OK) == -1) { + log_fatal("ziterate", + "unable to read specified allowlist file (%s)", + conf.allowlist_filename); + } + + // parse blocklist and allowlist + if (blocklist_init(conf.allowlist_filename, conf.blocklist_filename, + conf.destination_cidrs, conf.destination_cidrs_len, + NULL, 0, conf.ignore_errors)) { + log_fatal("ziterate", + "unable to initialize blocklist / allowlist"); + } + + // Set up sharding + conf.shard_num = 0; + conf.total_shards = 1; + if ((args.shard_given || args.shards_given) && !args.seed_given) { + log_fatal("ziterate", + "Need to specify seed if sharding a scan"); + } + if (args.shard_given ^ args.shards_given) { + log_fatal( + "ziterate", + "Need to specify both shard number and total number of shards"); + } + if (args.shard_given) { + enforce_range("shard", args.shard_arg, 0, 65534); + conf.shard_num = args.shard_arg; + } + if (args.shards_given) { + enforce_range("shards", args.shards_arg, 1, 65535); + conf.total_shards = args.shards_arg; + } + if (conf.shard_num >= conf.total_shards) { + log_fatal("ziterate", + "With %hhu total shards, shard number (%hhu)" + " must be in range [0, %hhu)", + conf.total_shards, conf.shard_num, conf.total_shards); + } + log_debug( + "ziterate", + "Initializing sharding (%d shards, shard number %d, seed %llu)", + conf.total_shards, conf.shard_num, conf.seed); + + // Check for a random seed + if (args.seed_given) { + conf.seed = args.seed_arg; + } else { + if (!random_bytes(&conf.seed, sizeof(uint64_t))) { + log_fatal("ziterate", "unable to generate random bytes " + "needed for seed"); + } + } + zconf.aes = aesrand_init_from_seed(conf.seed); + + zconf.ports = xmalloc(sizeof(struct port_conf)); + if (args.target_ports_given) { + parse_ports(args.target_ports_arg, zconf.ports); + } else { + zconf.ports->port_count = 1; + } + + uint64_t num_addrs = blocklist_count_allowed(); + if (zconf.list_of_ips_filename) { + log_debug("send", + "forcing max group size for compatibility with -I"); + num_addrs = 0xFFFFFFFF; + } + iterator_t *it = iterator_init(1, conf.shard_num, conf.total_shards, + num_addrs, zconf.ports->port_count); + shard_t *shard = get_shard(it, 0); + + target_t current = shard_get_cur_target(shard); + for (uint32_t count = 0; current.ip; ++count) { + if (conf.max_hosts && count >= conf.max_hosts) { + break; + } + struct in_addr next_ip; + next_ip.s_addr = current.ip; + if (current.port) { + printf("%s,%u\n", inet_ntoa(next_ip), current.port); + } else { + printf("%s\n", inet_ntoa(next_ip)); + } + current = shard_get_next_target(shard); + } + return EXIT_SUCCESS; +} diff --git a/src/zitopt.ggo.in b/src/zitopt.ggo.in new file mode 100644 index 0000000..4590c42 --- /dev/null +++ b/src/zitopt.ggo.in @@ -0,0 +1,59 @@ +# ZIterate Copyright 2014 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# ziterate option description to be processed by gengetopt + +package "ziterate" +version "@ZMAP_VERSION@" +purpose "A tool for iterating over the IPv4 space" + +section "Basic arguments" + +option "target-ports" p "comma-delimited list of ports to scan (for TCP and UDP scans)" + typestr="ports" + optional string +option "blocklist-file" b "File of subnets to exclude, in CIDR notation, one-per line." + optional string +option "allowlist-file" w "File of subnets to include, in CIDR notation, one-per line." + optional string +option "log-file" l "File to log to" + optional string +option "verbosity" v "Set log level verbosity (0-5, default 3)" + default="3" + optional int +option "ignore-blocklist-errors" - "Ignore invalid entries in the blocklist/allowlist (default false)" + optional +option "seed" e "Seed used to select address permutation" + typestr="n" + optional longlong +option "max-targets" n "Cap number of IPs to generate (as a number or a percentage of the address space)" + typestr="n" + optional string +option "disable-syslog" - "Disables logging messages to syslog" + optional + +section "Sharding" + +option "shards" - "total number of shards" + typestr="N" + optional int + default="1" +option "shard" - "shard this scan is targeting (0 indexed)" + typestr="n" + optional int + default="0" + +section "Additional options" + +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +text "\nExamples:\n\ + ziterate (iterate over all public IPv4 addresses)\n\ + ziterate -b exclusions 10.0.0.0/8 (iterate all IPs in 10./8 except those in blocklist)\n\ + ziterate -p 80,100-102 (scan full IPv4 on ports 80, 100, 101, 102)\n" diff --git a/src/zitopt_compat.c b/src/zitopt_compat.c new file mode 100644 index 0000000..5ba6275 --- /dev/null +++ b/src/zitopt_compat.c @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "zitopt.c" diff --git a/src/zmap.1 b/src/zmap.1 new file mode 100644 index 0000000..76c6bab --- /dev/null +++ b/src/zmap.1 @@ -0,0 +1,294 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "ZMAP" "1" "December 2023" "ZMap" "zmap" +. +.SH "NAME" +\fBzmap\fR \- The Fast Internet Scanner +. +.SH "SYNOPSIS" +zmap [ \-p ] [ \-o ] [ OPTIONS\.\.\. ] [ ip/hostname/range ] +. +.SH "DESCRIPTION" +\fIZMap\fR is a network tool for scanning the entire IPv4 address space (or large samples)\. ZMap is capable of scanning the entire Internet in around 45 minutes on a gigabit network connection, reaching ~98% theoretical line speed\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fBip\fR/\fBhostname\fR/\fBrange\fR +IP addresses or DNS hostnames to scan\. Accepts IP ranges in CIDR block notation\. Defaults to 0\.0\.0/8 +. +.TP +\fB\-p\fR, \fB\-\-target\-ports=port(s)\fR +List of TCP/UDP ports and/or port ranges to scan (e\.g\., 80,443,100\-105)\. Use \'*\' to scan all ports, including port 0\. +. +.TP +\fB\-o\fR, \fB\-\-output\-file=name\fR +When using an output module that uses a file, write results to this file\. Use \- for stdout\. +. +.TP +\fB\-b\fR, \fB\-\-blocklist\-file=path\fR +File of subnets to exclude, in CIDR notation, one\-per line\. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special\-purpose addresses\. An example blocklist file \fBblocklist\.conf\fR for this purpose\. +. +.TP +\fB\-w\fR, \fB\-\-allowlist\-file=path\fR +File of subnets to scan, in CIDR notation, one\-per line\. Specifying a allowlist file is equivalent to specifying to ranges directly on the command line interface, but allows specifying a large number of subnets\. Note: if you are specifying a large number of individual IP addresses (more than 10 million), you should instead use \fB\-\-list\-of\-ips\-file\fR\. +. +.TP +\fB\-I\fR, \fB\-\-list\-of\-ips\-file=path\fR +File of individual IP addresses to scan, one\-per line\. This feature allows you to scan a large number of unrelated addresses\. If you have a small number of IPs, it is faster to specify these on the command line or by using \fB\-\-allowlist\-file\fR\. This should only be used when scanning more than 10 million addresses\. When used in with \-\-allowlist\-path, only hosts in the intersection of both sets will be scanned\. Hosts specified here, but included in the blocklist will be excluded\. +. +.SS "SCAN OPTIONS" +. +.TP +\fB\-r\fR, \fB\-\-rate=pps\fR +Set the send rate in packets/sec\. Note: when combined with \-\-probes, this is total packets per second, not IPs per second\. Setting the rate to 0 will scan at full line rate\. Default: 10000 pps\. +. +.TP +\fB\-B\fR, \fB\-\-bandwidth=bps\fR +Set the send rate in bits/second (supports suffixes G, M, and K (e\.g\. \-B 10M for 10 mbps)\. This overrides the \-\-rate flag\. +. +.TP +\fB\-n\fR, \fB\-\-max\-targets=n\fR +Cap the number of targets to probe\. This can either be a number (e\.g\. \-n 1000) or a percentage (e\.g\. \-n 0\.1%) of the scannable address space (after excluding blocklist) +. +.TP +\fB\-N\fR, \fB\-\-max\-results=n\fR +Exit after receiving this many results +. +.TP +\fB\-t\fR, \fB\-\-max\-runtime=secs\fR +Cap the length of time for sending packets +. +.TP +\fB\-c\fR, \fB\-\-cooldown\-time=secs\fR +How long to continue receiving after sending has completed (default=8) +. +.TP +\fB\-e\fR, \fB\-\-seed=n\fR +Seed used to select address permutation\. Use this if you want to scan addresses in the same order for multiple ZMap runs\. +. +.TP +\fB\-P\fR, \fB\-\-probes=n\fR +Number of probes to send to each IP/Port pair (default=1)\. Since ZMap composes Ethernet frames directly, probes can be lost en\-route to destination\. Increasing the \-\-probes increases the chance that an online host will receive a probe in an unreliable network\. This is contrasted with \fB\-\-retries\fR which just gives the number of attempts to send a single probe on the source NIC\. +. +.TP +\fB\-\-retries=n\fR +Number of times to try resending a packet if the sendto call fails (default=10) +. +.TP +\fB\-\-batch=n\fR +Number of packets to batch before calling the appropriate syscall to send\. Used to take advantage of Linux\'s \fBsendmmsg\fR syscall to send the entire batch at once\. Only available on Linux, other OS\'s will send each packet individually\. (default=64) +. +.SS "SCAN SHARDING" +. +.TP +\fB\-\-shards=N\fR +Split the scan up into N shards/partitions among different instances of zmap (default=1)\. When sharding, \fB\-\-seed\fR is required\. +. +.TP +\fB\-\-shard=n\fR +Set which shard to scan (default=0)\. Shards are 0\-indexed in the range [0, N), where N is the total number of shards\. When sharding \fB\-\-seed\fR is required\. +. +.SS "NETWORK OPTIONS" +. +.TP +\fB\-s\fR, \fB\-\-source\-port=port|range\fR +Source port(s) to send packets from +. +.TP +\fB\-S\fR, \fB\-\-source\-ip=ip|range\fR +Source address(es) to send packets from\. Either single IP or range (e\.g\. 10\.0\.0\.1\-10\.0\.0\.9) +. +.TP +\fB\-G\fR, \fB\-\-gateway\-mac=addr\fR +Gateway MAC address to send packets to (in case auto\-detection fails) +. +.TP +\fB\-\-source\-mac=addr\fR +Source MAC address to send packets from (in case auto\-detection fails) +. +.TP +\fB\-i\fR, \fB\-\-interface=name\fR +Network interface to use +. +.TP +\fB\-X\fR, \fB\-\-iplayer\fR +Send IP layer packets instead of ethernet packets (for non\-Ethernet interface) +. +.SS "PROBE OPTIONS" +ZMap allows users to specify and write their own probe modules\. Probe modules are responsible for generating probe packets to send, and processing responses from hosts\. +. +.TP +\fB\-\-list\-probe\-modules\fR +List available probe modules (e\.g\. tcp_synscan) +. +.TP +\fB\-M\fR, \fB\-\-probe\-module=name\fR +Select probe module (default=tcp_synscan) +. +.TP +\fB\-\-probe\-args=args\fR +Arguments to pass to probe module +. +.TP +\fB\-\-probe\-ttl=hops\fR +Set TTL value for probe IP packets +. +.TP +\fB\-\-list\-output\-fields\fR +List the fields the selected probe module can send to the output module +. +.SS "OUTPUT OPTIONS" +ZMap allows users to specify and write their own output modules for use with ZMap\. Output modules are responsible for processing the fieldsets returned by the probe module, and outputting them to the user\. Users can specify output fields, and write filters over the output fields\. +. +.TP +\fB\-\-list\-output\-modules\fR +List available output modules (e\.g\. csv) +. +.TP +\fB\-O\fR, \fB\-\-output\-module=name\fR +Select output module (default=csv) +. +.TP +\fB\-\-output\-args=args\fR +Arguments to pass to output module +. +.TP +\fB\-f\fR, \fB\-\-output\-fields=fields\fR +Comma\-separated list of fields to output +. +.TP +\fB\-\-output\-filter\fR +Specify an output filter over the fields defined by the probe module\. See the output filter section for more details\. +. +.TP +\fB\-\-no\-header\-row\fR +Excludes any header rows (e\.g\., CSV header fields) from ZMap output\. This is useful if you\'re piping results into another application that expects only data\. +. +.SS "RESPONSE DEDUPLICATION" +Hosts will oftentimes send multiple responses to a probe (either because the scanner doesn\'t send back a RST packet or because the host has a misimplemented TCP stack\. To address this, ZMap will attempt to deduplicate responsive (ip,port) targets\. +. +.TP +\fB\-\-dedup\-method\fR +Specifies the method ZMap will use to deduplicate responses\. Options are: full, window, and none\. Full deduplication uses a 32\-bit bitmap and guarantees that no duplicates will be emitted\. However, full\-deduplication requires around 500MB of memory for a single port\. We do not support full deduplication for multiple ports\. Window uses a sliding window of a the last (user\-defined) number of responses as set by \-\-dedup\-window\-size\. None will prevent any deduplication\. +. +.TP +\fB\-\-dedup\-window\-size=targets\fR +Specifies the size of the sliding window of last n target responses to be used for deduplication\. Only applicable if using window deduplication\. +. +.SS "LOGGING AND METADATA OPTIONS" +. +.TP +\fB\-q\fR, \fB\-\-quiet\fR +Do not print status updates once per second +. +.TP +\fB\-v\fR, \fB\-\-verbosity=n\fR +Level of log detail (0\-5, default=3) +. +.TP +\fB\-l\fR, \fB\-\-log\-file=filename\fR +Output file for log messages\. By default, stderr\. +. +.TP +\fB\-m\fR, \fB\-\-metadata\-file=filename\fR +Output file for scan metadata (JSON) +. +.TP +\fB\-L\fR, \fB\-\-log\-directory\fR +Write log entries to a timestamped file in this directory +. +.TP +\fB\-u\fR, \fB\-\-status\-updates\-file\fR +Write scan progress updates to CSV file" +. +.TP +\fB\-\-disable\-syslog\fR +Disables logging messages to syslog +. +.TP +\fB\-\-notes\fR +Inject user\-specified notes into scan metadata +. +.TP +\fB\-\-user\-metadata\fR +Inject user\-specified JSON metadata into scan metadata +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-T\fR, \fB\-\-sender\-threads=n\fR +Threads used to send packets\. ZMap will attempt to detect the optimal number of send threads based on the number of processor cores\. Defaults to min(4, number of processor cores on host \- 1)\. +. +.TP +\fB\-C\fR, \fB\-\-config=filename\fR +Read a configuration file, which can specify any other options\. +. +.TP +\fB\-d\fR, \fB\-\-dryrun\fR +Print out each packet to stdout instead of sending it (useful for debugging) +. +.TP +\fB\-\-max\-sendto\-failures\fR +Maximum NIC sendto failures before scan is aborted +. +.TP +\fB\-\-min\-hitrate\fR +Minimum hitrate that scan can hit before scan is aborted +. +.TP +\fB\-\-cores\fR +Comma\-separated list of cores to pin to +. +.TP +\fB\-\-ignore\-blocklist\-errors\fR +Ignore invalid, malformed, or unresolvable entries in allowlist/blocklist file\. Replaces the pre\-v3\.x \fB\-\-ignore\-invalid\-hosts\fR option\. +. +.TP +\fB\-h\fR, \fB\-\-help\fR +Print help and exit +. +.TP +\fB\-V\fR, \fB\-\-version\fR +Print version and exit +. +.SS "OUTPUT FILTERS" +Results generated by a probe module can be filtered before being passed to the output module\. Filters are defined over the output fields of a probe module\. Filters are written in a simple filtering language, similar to SQL, and are passed to ZMap using the \fB\-\-output\-filter\fR option\. Output filters are commonly used to filter out duplicate results, or to only pass only successful responses to the output module\. +. +.P +Filter expressions are of the form \fB \fR\. The type of \fB\fR must be either a string or unsigned integer literal, and match the type of \fB\fR\. The valid operations for integer comparisons are = !=, \fI,\fR, \fI=,\fR=\. The operations for string comparisons are =, !=\. The \fB\-\-list\-output\-fields\fR flag will print what fields and types are available for the selected probe module, and then exit\. +. +.P +Compound filter expressions may be constructed by combining filter expressions using parenthesis to specify order of operations, the && (logical AND) and || (logical OR) operators\. +. +.P +For example, a filter for only successful, non\-duplicate responses would be written as: \fB\-\-output\-filter="success = 1 && repeat = 0"\fR +. +.SS "UDP PROBE MODULE OPTIONS" +These arguments are all passed using the \fB\-\-probe\-args=args\fR option\. Only one argument may be passed at a time\. +. +.TP +\fBfile:/path/to/file\fR +Path to payload file to send to each host over UDP\. +. +.TP +\fBtemplate:/path/to/template\fR +Path to template file\. For each destination host, the template file is populated, set as the UDP payload, and sent\. +. +.TP +\fBtext:\fR +ASCII text to send to each destination host +. +.TP +\fBhex:\fR +Hex\-encoded binary to send to each destination host +. +.TP +\fBtemplate\-fields\fR +Print information about the allowed template fields and exit\. +. +.SS "MID\-SCAN CHANGES" +You can change the rate at which ZMap is scanning mid\-scan by sending SIGUSR1 (increase) and SIGUSR2 (decrease) signals to ZMap\. These will result in the scan rate increasing or decreasing by 5%\. diff --git a/src/zmap.1.html b/src/zmap.1.html new file mode 100644 index 0000000..6740e60 --- /dev/null +++ b/src/zmap.1.html @@ -0,0 +1,307 @@ + + + + + + zmap(1) - The Fast Internet Scanner + + + + +
+ + + +
    +
  1. zmap(1)
    2. +
  2. zmap
    3. +
  3. zmap(1)
    4. +
+ +

NAME

+

+ zmap - The Fast Internet Scanner +

+ +

SYNOPSIS

+ +

zmap [ -p <port(s)> ] [ -o <outfile> ] [ OPTIONS... ] [ ip/hostname/range ]

+ +

DESCRIPTION

+ +

ZMap is a network tool for scanning the entire IPv4 address space (or large +samples). ZMap is capable of scanning the entire Internet in around 45 minutes +on a gigabit network connection, reaching ~98% theoretical line speed.

+ +

OPTIONS

+ +

BASIC OPTIONS

+ +
+
ip/hostname/range

IP addresses or DNS hostnames to scan. Accepts IP ranges in CIDR block + notation. Defaults to 0.0.0/8

+
-p, --target-ports=port(s)

List of TCP/UDP ports and/or port ranges to scan (e.g., 80,443,100-105). + Use '*' to scan all ports, including port 0.

+
-o, --output-file=name

When using an output module that uses a file, write results to this file. + Use - for stdout.

+
-b, --blocklist-file=path

File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file blocklist.conf for this purpose.

+
-w, --allowlist-file=path

File of subnets to scan, in CIDR notation, one-per line. Specifying a +allowlist file is equivalent to specifying to ranges directly on the command +line interface, but allows specifying a large number of subnets. Note: +if you are specifying a large number of individual IP addresses (more than +10 million), you should instead use --list-of-ips-file.

+
-I, --list-of-ips-file=path

File of individual IP addresses to scan, one-per line. This feature allows you +to scan a large number of unrelated addresses. If you have a small number of IPs, +it is faster to specify these on the command +line or by using --allowlist-file. This should only be used when scanning more than +10 million addresses. When used in with --allowlist-path, only hosts in the intersection +of both sets will be scanned. Hosts specified here, but included in the blocklist will +be excluded.

+
+ + +

SCAN OPTIONS

+ +
+
-r, --rate=pps

Set the send rate in packets/sec. Note: when combined with --probes, this is + total packets per second, not IPs per second. Setting the rate to 0 will scan + at full line rate. Default: 10000 pps.

+
-B, --bandwidth=bps

Set the send rate in bits/second (supports suffixes G, M, and K (e.g. -B + 10M for 10 mbps). This overrides the --rate flag.

+
-n, --max-targets=n

Cap the number of targets to probe. This can either be a number (e.g. -n + 1000) or a percentage (e.g. -n 0.1%) of the scannable address space + (after excluding blocklist)

+
-N, --max-results=n

Exit after receiving this many results

+
-t, --max-runtime=secs

Cap the length of time for sending packets

+
-c, --cooldown-time=secs

How long to continue receiving after sending has completed (default=8)

+
-e, --seed=n

Seed used to select address permutation. Use this if you want to scan + addresses in the same order for multiple ZMap runs.

+
-P, --probes=n

Number of probes to send to each IP/Port pair (default=1). Since ZMap composes Ethernet + frames directly, probes can be lost en-route to destination. Increasing the + --probes increases the chance that an online host will receive a probe in an + unreliable network. This is contrasted with --retries which just gives the + number of attempts to send a single probe on the source NIC.

+
--retries=n

Number of times to try resending a packet if the sendto call fails (default=10)

+
--batch=n

Number of packets to batch before calling the appropriate syscall to send. Used + to take advantage of Linux's sendmmsg syscall to send the entire batch at once. + Only available on Linux, other OS's will send each packet individually. (default=64)

+
+ + +

SCAN SHARDING

+ +
+
--shards=N

Split the scan up into N shards/partitions among different instances of + zmap (default=1). When sharding, --seed is required.

+
--shard=n

Set which shard to scan (default=0). Shards are 0-indexed in the range + [0, N), where N is the total number of shards. When sharding + --seed is required.

+
+ + +

NETWORK OPTIONS

+ +
+
-s, --source-port=port|range

Source port(s) to send packets from

+
-S, --source-ip=ip|range

Source address(es) to send packets from. Either single IP or range (e.g. + 10.0.0.1-10.0.0.9)

+
-G, --gateway-mac=addr

Gateway MAC address to send packets to (in case auto-detection fails)

+
--source-mac=addr

Source MAC address to send packets from (in case auto-detection fails)

+
-i, --interface=name

Network interface to use

+
-X, --iplayer

Send IP layer packets instead of ethernet packets (for non-Ethernet interface)

+
+ + +

PROBE OPTIONS

+ +

ZMap allows users to specify and write their own probe modules. Probe modules +are responsible for generating probe packets to send, and processing responses +from hosts.

+ +
+
--list-probe-modules

List available probe modules (e.g. tcp_synscan)

+
-M, --probe-module=name

Select probe module (default=tcp_synscan)

+
--probe-args=args

Arguments to pass to probe module

+
--probe-ttl=hops

Set TTL value for probe IP packets

+
--list-output-fields

List the fields the selected probe module can send to the output module

+
+ + +

OUTPUT OPTIONS

+ +

ZMap allows users to specify and write their own output modules for use with +ZMap. Output modules are responsible for processing the fieldsets returned by +the probe module, and outputting them to the user. Users can specify output +fields, and write filters over the output fields.

+ +
+
--list-output-modules

List available output modules (e.g. csv)

+
-O, --output-module=name

Select output module (default=csv)

+
--output-args=args

Arguments to pass to output module

+
-f, --output-fields=fields

Comma-separated list of fields to output

+
--output-filter

Specify an output filter over the fields defined by the probe module. See + the output filter section for more details.

+
--no-header-row

Excludes any header rows (e.g., CSV header fields) from ZMap output. This is + useful if you're piping results into another application that expects only + data.

+
+ + +

RESPONSE DEDUPLICATION

+ +

Hosts will oftentimes send multiple responses to a probe (either because the +scanner doesn't send back a RST packet or because the host has a misimplemented +TCP stack. To address this, ZMap will attempt to deduplicate responsive (ip,port) +targets.

+ +
+
--dedup-method

Specifies the method ZMap will use to deduplicate responses. Options are: + full, window, and none. Full deduplication uses a 32-bit bitmap and + guarantees that no duplicates will be emitted. However, full-deduplication + requires around 500MB of memory for a single port. We do not support full + deduplication for multiple ports. Window uses a sliding window of a the last + (user-defined) number of responses as set by --dedup-window-size. None will + prevent any deduplication.

+
--dedup-window-size=targets

Specifies the size of the sliding window of last n target responses to be + used for deduplication. Only applicable if using window deduplication.

+
+ + +

LOGGING AND METADATA OPTIONS

+ +
+
-q, --quiet

Do not print status updates once per second

+
-v, --verbosity=n

Level of log detail (0-5, default=3)

+
-l, --log-file=filename

Output file for log messages. By default, stderr.

+
-m, --metadata-file=filename

Output file for scan metadata (JSON)

+
-L, --log-directory

Write log entries to a timestamped file in this directory

+
-u, --status-updates-file

Write scan progress updates to CSV file"

+
--disable-syslog

Disables logging messages to syslog

+
--notes

Inject user-specified notes into scan metadata

+
--user-metadata

Inject user-specified JSON metadata into scan metadata

+
+ + +

ADDITIONAL OPTIONS

+ +
+
-T, --sender-threads=n

Threads used to send packets. ZMap will attempt to detect the optimal + number of send threads based on the number of processor cores. Defaults to + min(4, number of processor cores on host - 1).

+
-C, --config=filename

Read a configuration file, which can specify any other options.

+
-d, --dryrun

Print out each packet to stdout instead of sending it (useful for + debugging)

+
--max-sendto-failures

Maximum NIC sendto failures before scan is aborted

+
--min-hitrate

Minimum hitrate that scan can hit before scan is aborted

+
--cores

Comma-separated list of cores to pin to

+
--ignore-blocklist-errors

Ignore invalid, malformed, or unresolvable entries in allowlist/blocklist file. + Replaces the pre-v3.x --ignore-invalid-hosts option.

+
-h, --help

Print help and exit

+
-V, --version

Print version and exit

+
+ + +

OUTPUT FILTERS

+ +

Results generated by a probe module can be filtered before being passed to the +output module. Filters are defined over the output fields of a probe module. +Filters are written in a simple filtering language, similar to SQL, and are +passed to ZMap using the --output-filter option. Output filters are commonly +used to filter out duplicate results, or to only pass only successful responses +to the output module.

+ +

Filter expressions are of the form <fieldname> <operation> <value>. The type of +<value> must be either a string or unsigned integer literal, and match the type +of <fieldname>. The valid operations for integer comparisons are = !=, , , +=, =. The operations for string comparisons are =, !=. The +--list-output-fields flag will print what fields and types are available for +the selected probe module, and then exit.

+ +

Compound filter expressions may be constructed by combining filter expressions +using parenthesis to specify order of operations, the && (logical AND) and || +(logical OR) operators.

+ +

For example, a filter for only successful, non-duplicate responses would be +written as: --output-filter="success = 1 && repeat = 0"

+ +

UDP PROBE MODULE OPTIONS

+ +

These arguments are all passed using the --probe-args=args option. Only one +argument may be passed at a time.

+ +
+
file:/path/to/file

Path to payload file to send to each host over UDP.

+
template:/path/to/template

Path to template file. For each destination host, the template file is + populated, set as the UDP payload, and sent.

+
text:<text>

ASCII text to send to each destination host

+
hex:<hex>

Hex-encoded binary to send to each destination host

+
template-fields

Print information about the allowed template fields and exit.

+
+ + +

MID-SCAN CHANGES

+ +

You can change the rate at which ZMap is scanning mid-scan by sending SIGUSR1 (increase) +and SIGUSR2 (decrease) signals to ZMap. These will result in the scan rate increasing or +decreasing by 5%.

+ + +
    +
  1. ZMap
    2. +
  2. December 2023
    3. +
  3. zmap(1)
    4. +
+ +
+ + diff --git a/src/zmap.1.ronn b/src/zmap.1.ronn new file mode 100644 index 0000000..6c9861a --- /dev/null +++ b/src/zmap.1.ronn @@ -0,0 +1,309 @@ +zmap(1) - The Fast Internet Scanner +=================================== + +## SYNOPSIS + +zmap [ -p <port(s)> ] [ -o <outfile> ] [ OPTIONS... ] [ ip/hostname/range ] + +## DESCRIPTION + +*ZMap* is a network tool for scanning the entire IPv4 address space (or large +samples). ZMap is capable of scanning the entire Internet in around 45 minutes +on a gigabit network connection, reaching ~98% theoretical line speed. + +## OPTIONS + +### BASIC OPTIONS ### + + * `ip`/`hostname`/`range`: + IP addresses or DNS hostnames to scan. Accepts IP ranges in CIDR block + notation. Defaults to 0.0.0/8 + + * `-p`, `--target-ports=port(s)`: + List of TCP/UDP ports and/or port ranges to scan (e.g., 80,443,100-105). + Use '*' to scan all ports, including port 0. + + * `-o`, `--output-file=name`: + When using an output module that uses a file, write results to this file. + Use - for stdout. + + * `-b`, `--blocklist-file=path`: + File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file **blocklist.conf** for this purpose. + + * `-w`, `--allowlist-file=path`: + File of subnets to scan, in CIDR notation, one-per line. Specifying a + allowlist file is equivalent to specifying to ranges directly on the command + line interface, but allows specifying a large number of subnets. Note: + if you are specifying a large number of individual IP addresses (more than + 10 million), you should instead use `--list-of-ips-file`. + + * `-I`, `--list-of-ips-file=path`: + File of individual IP addresses to scan, one-per line. This feature allows you + to scan a large number of unrelated addresses. If you have a small number of IPs, + it is faster to specify these on the command + line or by using `--allowlist-file`. This should only be used when scanning more than + 10 million addresses. When used in with --allowlist-path, only hosts in the intersection + of both sets will be scanned. Hosts specified here, but included in the blocklist will + be excluded. + +### SCAN OPTIONS ### + + * `-r`, `--rate=pps`: + Set the send rate in packets/sec. Note: when combined with --probes, this is + total packets per second, not IPs per second. Setting the rate to 0 will scan + at full line rate. Default: 10000 pps. + + * `-B`, `--bandwidth=bps`: + Set the send rate in bits/second (supports suffixes G, M, and K (e.g. -B + 10M for 10 mbps). This overrides the --rate flag. + + * `-n`, `--max-targets=n`: + Cap the number of targets to probe. This can either be a number (e.g. -n + 1000) or a percentage (e.g. -n 0.1%) of the scannable address space + (after excluding blocklist) + + * `-N`, `--max-results=n`: + Exit after receiving this many results + + * `-t`, `--max-runtime=secs`: + Cap the length of time for sending packets + + * `-c`, `--cooldown-time=secs`: + How long to continue receiving after sending has completed (default=8) + + * `-e`, `--seed=n`: + Seed used to select address permutation. Use this if you want to scan + addresses in the same order for multiple ZMap runs. + + * `-P`, `--probes=n`: + Number of probes to send to each IP/Port pair (default=1). Since ZMap composes Ethernet + frames directly, probes can be lost en-route to destination. Increasing the + --probes increases the chance that an online host will receive a probe in an + unreliable network. This is contrasted with `--retries` which just gives the + number of attempts to send a single probe on the source NIC. + + * `--retries=n`: + Number of times to try resending a packet if the sendto call fails (default=10) + + * `--batch=n`: + Number of packets to batch before calling the appropriate syscall to send. Used + to take advantage of Linux's `sendmmsg` syscall to send the entire batch at once. + Only available on Linux, other OS's will send each packet individually. (default=64) + +### SCAN SHARDING ### + + * `--shards=N`: + Split the scan up into N shards/partitions among different instances of + zmap (default=1). When sharding, **--seed** is required. + + * `--shard=n`: + Set which shard to scan (default=0). Shards are 0-indexed in the range + [0, N), where N is the total number of shards. When sharding + **--seed** is required. + +### NETWORK OPTIONS ### + + * `-s`, `--source-port=port|range`: + Source port(s) to send packets from + + * `-S`, `--source-ip=ip|range`: + Source address(es) to send packets from. Either single IP or range (e.g. + 10.0.0.1-10.0.0.9) + + * `-G`, `--gateway-mac=addr`: + Gateway MAC address to send packets to (in case auto-detection fails) + + * `--source-mac=addr`: + Source MAC address to send packets from (in case auto-detection fails) + + * `-i`, `--interface=name`: + Network interface to use + + * `-X`, `--iplayer`: + Send IP layer packets instead of ethernet packets (for non-Ethernet interface) + +### PROBE OPTIONS ### + +ZMap allows users to specify and write their own probe modules. Probe modules +are responsible for generating probe packets to send, and processing responses +from hosts. + + * `--list-probe-modules`: + List available probe modules (e.g. tcp_synscan) + + * `-M`, `--probe-module=name`: + Select probe module (default=tcp_synscan) + + * `--probe-args=args`: + Arguments to pass to probe module + + * `--probe-ttl=hops`: + Set TTL value for probe IP packets + + * `--list-output-fields`: + List the fields the selected probe module can send to the output module + +### OUTPUT OPTIONS ### + +ZMap allows users to specify and write their own output modules for use with +ZMap. Output modules are responsible for processing the fieldsets returned by +the probe module, and outputting them to the user. Users can specify output +fields, and write filters over the output fields. + + * `--list-output-modules`: + List available output modules (e.g. csv) + + * `-O`, `--output-module=name`: + Select output module (default=csv) + + * `--output-args=args`: + Arguments to pass to output module + + * `-f`, `--output-fields=fields`: + Comma-separated list of fields to output + + * `--output-filter`: + Specify an output filter over the fields defined by the probe module. See + the output filter section for more details. + + * `--no-header-row`: + Excludes any header rows (e.g., CSV header fields) from ZMap output. This is + useful if you're piping results into another application that expects only + data. + + +### RESPONSE DEDUPLICATION ### + +Hosts will oftentimes send multiple responses to a probe (either because the +scanner doesn't send back a RST packet or because the host has a misimplemented +TCP stack. To address this, ZMap will attempt to deduplicate responsive (ip,port) +targets. + + + * `--dedup-method`: + Specifies the method ZMap will use to deduplicate responses. Options are: + full, window, and none. Full deduplication uses a 32-bit bitmap and + guarantees that no duplicates will be emitted. However, full-deduplication + requires around 500MB of memory for a single port. We do not support full + deduplication for multiple ports. Window uses a sliding window of a the last + (user-defined) number of responses as set by --dedup-window-size. None will + prevent any deduplication. + + * `--dedup-window-size=targets`: + Specifies the size of the sliding window of last n target responses to be + used for deduplication. Only applicable if using window deduplication. + +### LOGGING AND METADATA OPTIONS ### + + * `-q`, `--quiet`: + Do not print status updates once per second + + * `-v`, `--verbosity=n`: + Level of log detail (0-5, default=3) + + * `-l`, `--log-file=filename`: + Output file for log messages. By default, stderr. + + * `-m`, `--metadata-file=filename`: + Output file for scan metadata (JSON) + + * `-L`, `--log-directory`: + Write log entries to a timestamped file in this directory + + * `-u`, `--status-updates-file`: + Write scan progress updates to CSV file" + + * `--disable-syslog`: + Disables logging messages to syslog + + * `--notes`: + Inject user-specified notes into scan metadata + + * `--user-metadata`: + Inject user-specified JSON metadata into scan metadata + +### ADDITIONAL OPTIONS ### + + * `-T`, `--sender-threads=n`: + Threads used to send packets. ZMap will attempt to detect the optimal + number of send threads based on the number of processor cores. Defaults to + min(4, number of processor cores on host - 1). + + * `-C`, `--config=filename`: + Read a configuration file, which can specify any other options. + + * `-d`, `--dryrun`: + Print out each packet to stdout instead of sending it (useful for + debugging) + + * `--max-sendto-failures`: + Maximum NIC sendto failures before scan is aborted + + * `--min-hitrate`: + Minimum hitrate that scan can hit before scan is aborted + + * `--cores`: + Comma-separated list of cores to pin to + + * `--ignore-blocklist-errors`: + Ignore invalid, malformed, or unresolvable entries in allowlist/blocklist file. + Replaces the pre-v3.x `--ignore-invalid-hosts` option. + + * `-h`, `--help`: + Print help and exit + + * `-V`, `--version`: + Print version and exit + +### OUTPUT FILTERS ### + +Results generated by a probe module can be filtered before being passed to the +output module. Filters are defined over the output fields of a probe module. +Filters are written in a simple filtering language, similar to SQL, and are +passed to ZMap using the `--output-filter` option. Output filters are commonly +used to filter out duplicate results, or to only pass only successful responses +to the output module. + +Filter expressions are of the form ` `. The type of +`` must be either a string or unsigned integer literal, and match the type +of ``. The valid operations for integer comparisons are = !=, <, >, +<=, >=. The operations for string comparisons are =, !=. The +`--list-output-fields` flag will print what fields and types are available for +the selected probe module, and then exit. + +Compound filter expressions may be constructed by combining filter expressions +using parenthesis to specify order of operations, the && (logical AND) and || +(logical OR) operators. + +For example, a filter for only successful, non-duplicate responses would be +written as: `--output-filter="success = 1 && repeat = 0"` + +### UDP PROBE MODULE OPTIONS ### + +These arguments are all passed using the `--probe-args=args` option. Only one +argument may be passed at a time. + + * `file:/path/to/file`: + Path to payload file to send to each host over UDP. + + * `template:/path/to/template`: + Path to template file. For each destination host, the template file is + populated, set as the UDP payload, and sent. + + * `text:`: + ASCII text to send to each destination host + + * `hex:`: + Hex-encoded binary to send to each destination host + + * `template-fields`: + Print information about the allowed template fields and exit. + +### MID-SCAN CHANGES ### + +You can change the rate at which ZMap is scanning mid-scan by sending SIGUSR1 (increase) +and SIGUSR2 (decrease) signals to ZMap. These will result in the scan rate increasing or +decreasing by 5%. diff --git a/src/zmap.c b/src/zmap.c new file mode 100644 index 0000000..95cb481 --- /dev/null +++ b/src/zmap.c @@ -0,0 +1,1060 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" +#include "../lib/pbm.h" + +#include "aesrand.h" +#include "constants.h" +#include "ports.h" +#include "zopt.h" +#include "send.h" +#include "recv.h" +#include "state.h" +#include "monitor.h" +#include "get_gateway.h" +#include "filter.h" +#include "summary.h" +#include "utility.h" + +#include "output_modules/output_modules.h" +#include "probe_modules/probe_modules.h" + +#ifdef PFRING +#include +static int32_t distrib_func(pfring_zc_pkt_buff *pkt, pfring_zc_queue *in_queue, + void *arg) +{ + (void)pkt; + (void)in_queue; + (void)arg; + return 0; +} +#endif + +pthread_mutex_t recv_ready_mutex = PTHREAD_MUTEX_INITIALIZER; + +int get_num_cores(void) { + return sysconf(_SC_NPROCESSORS_ONLN); +} + +typedef struct send_arg { + uint32_t cpu; + sock_t sock; + shard_t *shard; +} send_arg_t; + +typedef struct recv_arg { + uint32_t cpu; +} recv_arg_t; + +typedef struct mon_start_arg { + uint32_t cpu; + iterator_t *it; + pthread_mutex_t *recv_ready_mutex; +} mon_start_arg_t; + +const char *default_help_text = + "By default, ZMap prints out unique, successful " + "IP addresses (e.g., SYN-ACK from a TCP SYN scan) " + "in ASCII form (e.g., 192.168.1.5) to stdout or the specified output " + "file. Internally this is handled by the \"csv\" output module and is " + "equivalent to running zmap --output-module=csv --output-fields=saddr " + "--output-filter=\"success = 1 && repeat = 0\" --no-header-row."; + +static void *start_send(void *arg) +{ + send_arg_t *s = (send_arg_t *)arg; + log_debug("zmap", "Pinning a send thread to core %u", s->cpu); + set_cpu(s->cpu); + int ret = send_run(s->sock, s->shard); + free(s); + if (ret != EXIT_SUCCESS) { + log_fatal("send", "send_run failed, terminating"); + } + return NULL; +} + +static void *start_recv(void *arg) +{ + recv_arg_t *r = (recv_arg_t *)arg; + log_debug("zmap", "Pinning receive thread to core %u", r->cpu); + set_cpu(r->cpu); + recv_run(&recv_ready_mutex); + return NULL; +} + +static void *start_mon(void *arg) +{ + mon_start_arg_t *mon_arg = (mon_start_arg_t *)arg; + log_debug("zmap", "Pinning monitor thread to core %u", mon_arg->cpu); + set_cpu(mon_arg->cpu); + monitor_run(mon_arg->it, mon_arg->recv_ready_mutex); + free(mon_arg); + return NULL; +} + +static void start_zmap(void) +{ + if (zconf.iface == NULL) { + zconf.iface = get_default_iface(); + assert(zconf.iface); + log_debug("zmap", + "no interface provided. will use default" + " interface (%s).", + zconf.iface); + } + if (zconf.number_source_ips == 0) { + struct in_addr default_ip; + if (get_iface_ip(zconf.iface, &default_ip) < 0) { + log_fatal("zmap", + "could not detect default IP address for %s." + " Try specifying a source address (-S).", + zconf.iface); + } + zconf.source_ip_addresses[0] = default_ip.s_addr; + zconf.number_source_ips++; + log_debug( + "zmap", + "no source IP address given. will use default address: %s.", + inet_ntoa(default_ip)); + } + if (!zconf.gw_mac_set) { + struct in_addr gw_ip; + memset(&gw_ip, 0, sizeof(struct in_addr)); + if (get_default_gw(&gw_ip, zconf.iface) < 0) { + log_fatal( + "zmap", + "could not detect default gateway address for %s." + " Try setting default gateway mac address (-G)." + " If this is a newly launched machine, try completing an outgoing network connection (e.g. curl https://zmap.io), and trying again.", + zconf.iface); + } + log_debug("zmap", "found gateway IP %s on %s", inet_ntoa(gw_ip), + zconf.iface); + zconf.gw_ip = gw_ip.s_addr; + memset(&zconf.gw_mac, 0, MAC_ADDR_LEN); + if (get_hw_addr(&gw_ip, zconf.iface, zconf.gw_mac)) { + log_fatal( + "zmap", + "could not detect GW MAC address for %s on %s." + " Try setting default gateway mac address (-G), or run" + " \"arp \" in terminal." + " If this is a newly launched machine, try completing an outgoing network connection (e.g. curl https://zmap.io), and trying again.", + inet_ntoa(gw_ip), zconf.iface); + } + zconf.gw_mac_set = 1; + } + log_debug("send", "gateway MAC address %02x:%02x:%02x:%02x:%02x:%02x", + zconf.gw_mac[0], zconf.gw_mac[1], zconf.gw_mac[2], + zconf.gw_mac[3], zconf.gw_mac[4], zconf.gw_mac[5]); + // Initialization + assert(zconf.output_module && "no output module set"); + log_debug("zmap", "output module: %s", zconf.output_module->name); + if (zconf.output_module && zconf.output_module->init) { + if (zconf.output_module->init(&zconf, zconf.output_fields, + zconf.output_fields_len)) { + log_fatal( + "zmap", + "output module did not initialize successfully."); + } + } + + iterator_t *it = send_init(); + if (!it) { + log_fatal("zmap", "unable to initialize sending component"); + } + if (zconf.output_module && zconf.output_module->start) { + zconf.output_module->start(&zconf, &zsend, &zrecv); + } + + // start threads + uint32_t cpu = 0; + pthread_t *tsend, trecv, tmon; + int r; + if (!zconf.dryrun) { + recv_arg_t *recv_arg = xmalloc(sizeof(recv_arg_t)); + recv_arg->cpu = zconf.pin_cores[cpu % zconf.pin_cores_len]; + cpu += 1; + r = pthread_create(&trecv, NULL, start_recv, recv_arg); + if (r != 0) { + log_fatal("zmap", "unable to create recv thread"); + } + for (;;) { + pthread_mutex_lock(&recv_ready_mutex); + if (zconf.recv_ready) { + pthread_mutex_unlock(&recv_ready_mutex); + break; + } + pthread_mutex_unlock(&recv_ready_mutex); + } + } +#ifdef PFRING + pfring_zc_worker *zw = pfring_zc_run_balancer( + zconf.pf.queues, &zconf.pf.send, zconf.senders, 1, + zconf.pf.prefetches, round_robin_bursts_policy, NULL, distrib_func, + NULL, 0, zconf.pin_cores[cpu & zconf.pin_cores_len]); + cpu += 1; +#endif + tsend = xmalloc(zconf.senders * sizeof(pthread_t)); + for (uint8_t i = 0; i < zconf.senders; i++) { + sock_t sock; + if (zconf.dryrun) { + sock = get_dryrun_socket(); + } else { + sock = get_socket(i); + } + send_arg_t *arg = xmalloc(sizeof(send_arg_t)); + + arg->sock = sock; + arg->shard = get_shard(it, i); + arg->cpu = zconf.pin_cores[cpu % zconf.pin_cores_len]; + cpu += 1; + int r = pthread_create(&tsend[i], NULL, start_send, arg); + if (r != 0) { + log_fatal("zmap", "unable to create send thread"); + exit(EXIT_FAILURE); + } + } + log_debug("zmap", "%d sender threads spawned", zconf.senders); + + if (!zconf.dryrun) { + monitor_init(); + mon_start_arg_t *mon_arg = xmalloc(sizeof(mon_start_arg_t)); + mon_arg->it = it; + mon_arg->recv_ready_mutex = &recv_ready_mutex; + mon_arg->cpu = zconf.pin_cores[cpu % zconf.pin_cores_len]; + int r = pthread_create(&tmon, NULL, start_mon, mon_arg); + if (r != 0) { + log_fatal("zmap", "unable to create monitor thread"); + exit(EXIT_FAILURE); + } + } + +#ifndef PFRING + drop_privs(); +#endif + + // wait for completion + for (uint8_t i = 0; i < zconf.senders; i++) { + int r = pthread_join(tsend[i], NULL); + if (r != 0) { + log_fatal("zmap", "unable to join send thread"); + exit(EXIT_FAILURE); + } + } + log_debug("zmap", "senders finished"); +#ifdef PFRING + pfring_zc_kill_worker(zw); + pfring_zc_sync_queue(zconf.pf.send, tx_only); + log_debug("zmap", "send queue flushed"); +#endif + // no receiving or monitoring thread is started in dry run mode + if (!zconf.dryrun) { + r = pthread_join(trecv, NULL); + if (r != 0) { + log_fatal("zmap", "unable to join recv thread"); + exit(EXIT_FAILURE); + } + if (!zconf.quiet || zconf.status_updates_file) { + pthread_join(tmon, NULL); + if (r != 0) { + log_fatal("zmap", + "unable to join monitor thread"); + exit(EXIT_FAILURE); + } + } + } + + // finished + if (zconf.metadata_filename) { + json_metadata(zconf.metadata_file); + } + if (zconf.output_module && zconf.output_module->close) { + zconf.output_module->close(&zconf, &zsend, &zrecv); + } + if (zconf.probe_module && zconf.probe_module->close) { + zconf.probe_module->close(&zconf, &zsend, &zrecv); + } +#ifdef PFRING + pfring_zc_destroy_cluster(zconf.pf.cluster); +#endif + log_info("zmap", "completed"); +} + +#define SET_IF_GIVEN(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = args.ARG##_arg; \ + }; \ + } +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char *argv[]) +{ + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + int config_loaded = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + if (args.config_given || file_exists(args.config_arg)) { + params->initialize = 0; + params->override = 0; + if (cmdline_parser_config_file(args.config_arg, &args, + params) != 0) { + exit(EXIT_FAILURE); + } + config_loaded = 1; + } + + // set defaults before loading in command line arguments + init_empty_global_configuration(&zconf); + // initialize logging. if no log file or log directory are specified + // default to using stderr. + zconf.log_level = args.verbosity_arg; + zconf.log_file = args.log_file_arg; + zconf.log_directory = args.log_directory_arg; + if (args.disable_syslog_given) { + zconf.syslog = 0; + } else { + zconf.syslog = 1; + } + if (zconf.log_file && zconf.log_directory) { + log_init(stderr, zconf.log_level, zconf.syslog, "zmap"); + log_fatal("zmap", "log-file and log-directory cannot " + "specified simultaneously."); + } + FILE *log_location = NULL; + if (zconf.log_file) { + log_location = fopen(zconf.log_file, "w"); + } else if (zconf.log_directory) { + time_t now; + time(&now); + struct tm *local = localtime(&now); + char path[100]; + strftime(path, 100, "zmap-%Y-%m-%dT%H%M%S%z.log", local); + char *fullpath = + xmalloc(strlen(zconf.log_directory) + strlen(path) + 2); + sprintf(fullpath, "%s/%s", zconf.log_directory, path); + log_location = fopen(fullpath, "w"); + free(fullpath); + } else { + log_location = stderr; + } + if (!log_location) { + log_init(stderr, zconf.log_level, zconf.syslog, "zmap"); + log_fatal("zmap", "unable to open specified log file: %s", + strerror(errno)); + } + log_init(log_location, zconf.log_level, zconf.syslog, "zmap"); + log_debug("zmap", "zmap main thread started"); + if (config_loaded) { + log_debug("zmap", "Loaded configuration file %s", + args.config_arg); + } + if (zconf.syslog) { + log_debug("zmap", "syslog support enabled"); + } else { + log_info("zmap", "syslog support disabled"); + } + // parse the provided probe and output module s.t. that we can support + // other command-line helpers (e.g. probe help) + log_debug("zmap", "requested ouput-module: %s", args.output_module_arg); + + // ZMap's default behavior is to provide a simple file of the unique IP + // addresses that responded successfully. We only use this simple "default" + // mode if none of {output module, output filter, output fields} are set. + zconf.default_mode = + (!(args.output_module_given || args.output_filter_given || + args.output_fields_given)); + if (zconf.default_mode) { + log_info( + "zmap", + "By default, ZMap will output the unique IP addresses " + "of hosts that respond successfully (e.g., SYN-ACK packet). This " + "is equivalent to running ZMap with the following flags: " + "--output-module=csv --output-fields=saddr --output-filter='" + "success=1 && repeat=0' --no-header-row. " + "If you want all responses, explicitly set an output module or " + "set --output-filter=\"\"."); + zconf.output_module = get_output_module_by_name("csv"); + zconf.output_module_name = strdup("csv"); + zconf.no_header_row = 1; + } else if (!args.output_module_given) { + log_debug("zmap", "No output module provided. Will use csv."); + zconf.output_module = get_output_module_by_name("csv"); + zconf.output_module_name = strdup("csv"); + } else { + zconf.output_module = + get_output_module_by_name(args.output_module_arg); + if (!zconf.output_module) { + log_fatal( + "zmap", + "specified output module (%s) does not exist\n", + args.output_module_arg); + } + zconf.output_module_name = strdup(args.output_module_arg); + } + zconf.probe_module = get_probe_module_by_name(args.probe_module_arg); + if (!zconf.probe_module) { + log_fatal("zmap", + "specified probe module (%s) does not exist\n", + args.probe_module_arg); + exit(EXIT_FAILURE); + } + // check whether the probe module is going to generate dynamic data + // and that the output module can support exporting that data out of + // zmap. If they can't, then quit. + if (zconf.probe_module->output_type == OUTPUT_TYPE_DYNAMIC && + !zconf.output_module->supports_dynamic_output) { + log_fatal( + "zmap", + "specified probe module (%s) requires dynamic " + "output support, which output module (%s) does not support. " + "Most likely you want to use JSON output.", + args.probe_module_arg, args.output_module_arg); + } + if (args.help_given) { + cmdline_parser_print_help(); + printf("\nProbe Module (%s) Help:\n", zconf.probe_module->name); + if (zconf.probe_module->helptext) { + fprintw(stdout, zconf.probe_module->helptext, 80); + } else { + printf("no help text available\n"); + } + assert(zconf.output_module && "no output module set"); + const char *module_name = + zconf.default_mode ? "Default" : zconf.output_module->name; + printf("\nOutput Module (%s) Help:\n", module_name); + + if (zconf.default_mode) { + fprintw(stdout, default_help_text, 80); + } else if (zconf.output_module->helptext) { + fprintw(stdout, zconf.output_module->helptext, 80); + } else { + printf("no help text available\n"); + } + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + if (args.list_output_modules_given) { + print_output_modules(); + exit(EXIT_SUCCESS); + } + if (args.list_probe_modules_given) { + print_probe_modules(); + exit(EXIT_SUCCESS); + } + if (args.iplayer_given) { + zconf.send_ip_pkts = 1; + zconf.gw_mac_set = 1; + memset(zconf.gw_mac, 0, MAC_ADDR_LEN); + } + if (cmdline_parser_required(&args, CMDLINE_PARSER_PACKAGE) != 0) { + exit(EXIT_FAILURE); + } + + // now that we know the probe module, let's find what it supports + memset(&zconf.fsconf, 0, sizeof(struct fieldset_conf)); + // the set of fields made available to a user is constructed + // of IP header fields + probe module fields + system fields + fielddefset_t *fds = &(zconf.fsconf.defs); + gen_fielddef_set(fds, (fielddef_t *)&(ip_fields), ip_fields_len); + gen_fielddef_set(fds, zconf.probe_module->fields, + zconf.probe_module->numfields); + gen_fielddef_set(fds, (fielddef_t *)&(sys_fields), sys_fields_len); + if (args.list_output_fields_given) { + for (int i = 0; i < fds->len; i++) { + printf("%-15s %6s: %s\n", fds->fielddefs[i].name, + fds->fielddefs[i].type, fds->fielddefs[i].desc); + } + exit(EXIT_SUCCESS); + } + // find the fields we need for the framework + zconf.fsconf.success_index = fds_get_index_by_name(fds, "success"); + if (zconf.fsconf.success_index < 0) { + log_fatal("fieldset", "probe module does not supply " + "required success field."); + } + zconf.fsconf.app_success_index = + fds_get_index_by_name(fds, "app_success"); + + if (zconf.fsconf.app_success_index < 0) { + log_debug("fieldset", "probe module does not supply " + "application success field."); + } else { + log_debug( + "fieldset", + "probe module supplies app_success" + " output field. It will be included in monitor output"); + } + zconf.fsconf.classification_index = + fds_get_index_by_name(fds, "classification"); + if (zconf.fsconf.classification_index < 0) { + log_fatal("fieldset", "probe module does not supply " + "required packet classification field."); + } + zconf.ignore_invalid_hosts = args.ignore_blocklist_errors_given; + SET_BOOL(zconf.dryrun, dryrun); + SET_BOOL(zconf.quiet, quiet); + SET_BOOL(zconf.no_header_row, no_header_row); + zconf.cooldown_secs = args.cooldown_time_arg; + SET_IF_GIVEN(zconf.output_filename, output_file); + SET_IF_GIVEN(zconf.blocklist_filename, blocklist_file); + SET_IF_GIVEN(zconf.list_of_ips_filename, list_of_ips_file); + SET_IF_GIVEN(zconf.probe_args, probe_args); + SET_IF_GIVEN(zconf.probe_ttl, probe_ttl); + SET_IF_GIVEN(zconf.output_args, output_args); + SET_IF_GIVEN(zconf.iface, interface); + SET_IF_GIVEN(zconf.max_runtime, max_runtime); + SET_IF_GIVEN(zconf.max_results, max_results); + SET_IF_GIVEN(zconf.rate, rate); + SET_IF_GIVEN(zconf.packet_streams, probes); + SET_IF_GIVEN(zconf.status_updates_file, status_updates_file); + SET_IF_GIVEN(zconf.retries, retries); + SET_IF_GIVEN(zconf.max_sendto_failures, max_sendto_failures); + SET_IF_GIVEN(zconf.min_hitrate, min_hitrate); + + + if (zconf.retries < 0) { + log_fatal("zmap", "Invalid retry count"); + } + + if (zconf.max_sendto_failures >= 0) { + log_debug("zmap", + "scan will abort if more than %i " + "sendto failures occur", + zconf.max_sendto_failures); + } + if (zconf.min_hitrate > 0.0) { + log_debug("zmap", "scan will abort if hitrate falls below %f", + zconf.min_hitrate); + } + if (args.metadata_file_arg) { + zconf.metadata_filename = args.metadata_file_arg; + if (!strcmp(zconf.metadata_filename, "-")) { + zconf.metadata_file = stdout; + } else { + zconf.metadata_file = + fopen(zconf.metadata_filename, "w"); + } + if (!zconf.metadata_file) { + log_fatal("metadata", + "unable to open metadata file (%s): %s", + zconf.metadata_filename, strerror(errno)); + } + log_debug("metadata", "metadata will be saved to %s", + zconf.metadata_filename); + } + + if (args.user_metadata_given) { + zconf.custom_metadata_str = args.user_metadata_arg; + if (!json_tokener_parse(zconf.custom_metadata_str)) { + log_fatal("metadata", + "unable to parse custom user metadata"); + } else { + log_debug("metadata", + "user metadata validated successfully"); + } + } + if (args.notes_given) { + zconf.notes = args.notes_arg; + } + + // find if zmap wants any specific cidrs scanned instead + // of the entire Internet + zconf.destination_cidrs = args.inputs; + zconf.destination_cidrs_len = args.inputs_num; + if (zconf.destination_cidrs && zconf.blocklist_filename && + !strcmp(zconf.blocklist_filename, ZMAP_DEFAULT_BLOCKLIST)) { + log_warn( + "blocklist", + "ZMap is currently using the default blocklist located " + "at " ZMAP_DEFAULT_BLOCKLIST + ". By default, this blocklist excludes locally " + "scoped networks (e.g. 10.0.0.0/8, 127.0.0.1/8, and 192.168.0.0/16). If you are" + " trying to scan local networks, you can change the default blocklist by " + "editing the default ZMap configuration at " ZMAP_DEFAULT_BLOCKLIST + "." + " If you have modified the default blocklist, you can ignore this message."); + } + SET_IF_GIVEN(zconf.allowlist_filename, allowlist_file); + + if (zconf.probe_module->port_args) { + if (args.source_port_given) { + char *dash = strchr(args.source_port_arg, '-'); + if (dash) { // range + *dash = '\0'; + zconf.source_port_first = + atoi(args.source_port_arg); + enforce_range("starting source-port", + zconf.source_port_first, 0, + 0xFFFF); + zconf.source_port_last = atoi(dash + 1); + enforce_range("ending source-port", + zconf.source_port_last, 0, + 0xFFFF); + if (zconf.source_port_first > + zconf.source_port_last) { + fprintf( + stderr, + "%s: invalid source port range: " + "last port is less than first port\n", + CMDLINE_PARSER_PACKAGE); + exit(EXIT_FAILURE); + } + } else { // single port + int port = atoi(args.source_port_arg); + enforce_range("source-port", port, 0, 0xFFFF); + zconf.source_port_first = port; + zconf.source_port_last = port; + } + int num_source_ports = (zconf.source_port_last - zconf.source_port_first) + 1; + if (zconf.packet_streams > num_source_ports) { + log_fatal("zmap", "The number of probes sent to each target ip/port (%i) " + "must be smaller than the size of the source port range (%u-%u, size: %i). " + "Otherwise, some generated probe packets will be identical.", zconf.packet_streams, + zconf.source_port_first, zconf.source_port_last, + (zconf.source_port_last - zconf.source_port_first) + 1); + } else if (((float)zconf.packet_streams / (float)num_source_ports) < 0.1) { + log_warn("zmap", "ZMap is configured to use a relatively small number" + " of source ports (fewer than 10x the number of probe packets per target ip/port)," + " which limits the entropy that ZMap has available for " + " validating responses. We recommend that you use a larger port range."); + } + } + if (!args.target_ports_given) { + log_fatal("zmap", + "target ports (-p) required for %s probe", + zconf.probe_module->name); + } + } else { + if (args.target_ports_given) { + log_fatal("zmap", + "Destination port cannot be set for %s probe", + zconf.probe_module->name); + } + } + + zconf.ports = xmalloc(sizeof(struct port_conf)); + zconf.ports->port_bitmap = bm_init(); + if (args.target_ports_given) { + parse_ports(args.target_ports_arg, zconf.ports); + } else { + char *line = strdup("0"); + parse_ports(line, zconf.ports); + } + + if (args.dedup_method_given) { + if (!strcmp(args.dedup_method_arg, "default")) { + if (zconf.ports->port_count > 1) { + zconf.dedup_method = DEDUP_METHOD_WINDOW; + } else { + zconf.dedup_method = DEDUP_METHOD_FULL; + } + } else if (!strcmp(args.dedup_method_arg, "none")) { + zconf.dedup_method = DEDUP_METHOD_NONE; + } else if (!strcmp(args.dedup_method_arg, "full")) { + zconf.dedup_method = DEDUP_METHOD_FULL; + } else if (!strcmp(args.dedup_method_arg, "window")) { + zconf.dedup_method = DEDUP_METHOD_WINDOW; + } else { + log_fatal( + "dedup", + "Invalid dedup option provided. Legal options are: default, none, full, window."); + } + } else { + if (zconf.ports->port_count > 1) { + zconf.dedup_method = DEDUP_METHOD_WINDOW; + } else { + zconf.dedup_method = DEDUP_METHOD_FULL; + } + } + if (zconf.dedup_method == DEDUP_METHOD_FULL && + zconf.ports->port_count > 1) { + log_fatal( + "dedup", + "full response de-duplication is not supported for multiple ports"); + } + if (zconf.dedup_method == DEDUP_METHOD_WINDOW) { + if (args.dedup_window_size_given) { + zconf.dedup_window_size = args.dedup_window_size_arg; + } else { + zconf.dedup_window_size = 1000000; + } + log_info("dedup", + "Response deduplication method is %s with size %u", + DEDUP_METHOD_NAMES[zconf.dedup_method], + zconf.dedup_window_size); + } else { + log_info("dedup", "Response deduplication method is %s", + DEDUP_METHOD_NAMES[zconf.dedup_method]); + } + + // process the list of requested output fields. + if (args.output_fields_given) { + zconf.raw_output_fields = args.output_fields_arg; + } else { + if (zconf.ports->port_count > 1) { + zconf.raw_output_fields = "saddr,sport"; + } else { + zconf.raw_output_fields = "saddr"; + } + } + // add all fields if wildcard received + if (!strcmp(zconf.raw_output_fields, "*")) { + zconf.output_fields_len = zconf.fsconf.defs.len; + zconf.output_fields = + xcalloc(zconf.fsconf.defs.len, sizeof(const char *)); + for (int i = 0; i < zconf.fsconf.defs.len; i++) { + zconf.output_fields[i] = + zconf.fsconf.defs.fielddefs[i].name; + } + fs_generate_full_fieldset_translation(&zconf.fsconf.translation, + &zconf.fsconf.defs); + } else { + split_string(zconf.raw_output_fields, + &(zconf.output_fields_len), + &(zconf.output_fields)); + for (int i = 0; i < zconf.output_fields_len; i++) { + log_debug("zmap", "requested output field (%i): %s", i, + zconf.output_fields[i]); + } + // generate a translation that can be used to convert output + // from a probe module to the input for an output module + fs_generate_fieldset_translation( + &zconf.fsconf.translation, &zconf.fsconf.defs, + zconf.output_fields, zconf.output_fields_len); + } + + // default filtering behavior is to drop unsuccessful and duplicates + if (zconf.default_mode) { + log_debug( + "filter", + "No output filter specified. Will use default: exclude duplicates and unssuccessful"); + } else if (args.output_filter_given && + strcmp(args.output_filter_arg, "")) { + // Run it through yyparse to build the expression tree + if (!parse_filter_string(args.output_filter_arg)) { + log_fatal("zmap", "Unable to parse filter expression"); + } + // Check the fields used against the fieldset in use + if (!validate_filter(zconf.filter.expression, + &zconf.fsconf.defs)) { + log_fatal("zmap", "Invalid filter"); + } + zconf.output_filter_str = args.output_filter_arg; + log_debug("filter", "will use output filter %s", + args.output_filter_arg); + } else if (args.output_filter_given) { // (empty filter argument) + log_debug( + "filter", + "Empty output filter provided. ZMap will output all " + "results, including duplicate and non-successful responses."); + } else { + log_info( + "filter", + "No output filter provided. ZMap will output all " + "results, including duplicate and non-successful responses (e.g., " + "RST and ICMP packets). If you want a filter similar to ZMap's " + "default behavior, you can set an output filter similar to the " + "following: --output-filter=\"success=1 && repeat=0\"."); + } + + if (args.source_ip_given) { + parse_source_ip_addresses(args.source_ip_arg); + } + if (args.gateway_mac_given) { + if (!parse_mac(zconf.gw_mac, args.gateway_mac_arg)) { + fprintf(stderr, "%s: invalid MAC address `%s'\n", + CMDLINE_PARSER_PACKAGE, args.gateway_mac_arg); + exit(EXIT_FAILURE); + } + zconf.gw_mac_set = 1; + } + if (args.source_mac_given) { + if (!parse_mac(zconf.hw_mac, args.source_mac_arg)) { + fprintf(stderr, "%s: invalid MAC address `%s'\n", + CMDLINE_PARSER_PACKAGE, args.gateway_mac_arg); + exit(EXIT_FAILURE); + } + log_debug("send", + "source MAC address specified on CLI: " + "%02x:%02x:%02x:%02x:%02x:%02x", + zconf.hw_mac[0], zconf.hw_mac[1], zconf.hw_mac[2], + zconf.hw_mac[3], zconf.hw_mac[4], zconf.hw_mac[5]); + + zconf.hw_mac_set = 1; + } + // Check for a random seed + if (args.seed_given) { + zconf.seed = args.seed_arg; + zconf.seed_provided = 1; + } else { + // generate a seed randomly + if (!random_bytes(&zconf.seed, sizeof(uint64_t))) { + log_fatal("zmap", "unable to generate random bytes " + "needed for seed"); + } + zconf.seed_provided = 0; + } + zconf.aes = aesrand_init_from_seed(zconf.seed); + + // Set up sharding + zconf.shard_num = 0; + zconf.total_shards = 1; + if ((args.shard_given || args.shards_given) && !args.seed_given) { + log_fatal("zmap", "Need to specify seed if sharding a scan"); + } + if (args.shard_given ^ args.shards_given) { + log_fatal( + "zmap", + "Need to specify both shard number and total number of shards"); + } + if (args.shard_given) { + enforce_range("shard", args.shard_arg, 0, 65534); + } + if (args.shards_given) { + enforce_range("shards", args.shards_arg, 1, 65535); + } + SET_IF_GIVEN(zconf.shard_num, shard); + SET_IF_GIVEN(zconf.total_shards, shards); + if (zconf.shard_num >= zconf.total_shards) { + log_fatal("zmap", + "With %hhu total shards, shard number (%hhu)" + " must be in range [0, %hhu)", + zconf.total_shards, zconf.shard_num, + zconf.total_shards); + } + + if (args.bandwidth_given) { + // Supported: G,g=*1000000000; M,m=*1000000 K,k=*1000 bits per + // second + zconf.bandwidth = atoi(args.bandwidth_arg); + char *suffix = args.bandwidth_arg; + while (*suffix >= '0' && *suffix <= '9') { + suffix++; + } + if (*suffix) { + switch (*suffix) { + case 'G': + case 'g': + zconf.bandwidth *= 1000000000; + break; + case 'M': + case 'm': + zconf.bandwidth *= 1000000; + break; + case 'K': + case 'k': + zconf.bandwidth *= 1000; + break; + default: + fprintf(stderr, + "%s: unknown bandwidth suffix '%s' " + "(supported suffixes are G, M and K)\n", + CMDLINE_PARSER_PACKAGE, suffix); + exit(EXIT_FAILURE); + } + } + } + + if (args.batch_given && args.batch_arg >= 1 && args.batch_arg <= UINT8_MAX) { + zconf.batch = args.batch_arg; + } else if (args.batch_given) { + log_fatal("zmap", "batch size must be > 0 and <= 255"); + } + + if (args.max_targets_given) { + zconf.max_targets = parse_max_hosts(args.max_targets_arg); + } + + // blocklist + if (blocklist_init(zconf.allowlist_filename, zconf.blocklist_filename, + zconf.destination_cidrs, zconf.destination_cidrs_len, + NULL, 0, zconf.ignore_invalid_hosts)) { + log_fatal("zmap", "unable to initialize blocklist / allowlist"); + } + // if there's a list of ips to scan, then initialize PBM and populate + // it based on the provided file + if (zconf.list_of_ips_filename) { + zsend.list_of_ips_pbm = pbm_init(); + zconf.list_of_ips_count = pbm_load_from_file( + zsend.list_of_ips_pbm, zconf.list_of_ips_filename); + } + + // compute number of targets + uint64_t allowed = blocklist_count_allowed(); + zconf.total_allowed = allowed; + zconf.total_disallowed = blocklist_count_not_allowed(); + assert(allowed <= (1LL << 32)); + if (!zconf.total_allowed) { + log_fatal("zmap", "zero eligible addresses to scan"); + } + if (zconf.list_of_ips_count > 0 && + 0xFFFFFFFFU / zconf.list_of_ips_count > 100000) { + log_warn( + "zmap", + "list of IPs is small compared to address space. Performance will suffer, consider using an allowlist instead"); + } + if (zconf.max_targets) { + zsend.max_targets = zconf.max_targets; + } +#ifndef PFRING + // Set the correct number of threads, default to min(4, number of cores on host - 1, as available) + if (args.sender_threads_given) { + zconf.senders = args.sender_threads_arg; + } else { + // use one fewer than the number of cores on the machine such that the + // receiver thread can use a core for processing responses + int available_cores = get_num_cores(); + if (available_cores > 1) { + available_cores--; + } + int senders = min_int(available_cores, 4); + zconf.senders = senders; + log_debug("zmap", "will use %i sender threads based on core availability", senders); + } + if (2 * zconf.senders >= zsend.max_targets) { + log_warn( + "zmap", + "too few targets relative to senders, dropping to one sender"); + zconf.senders = 1; + } +#else + zconf.senders = args.sender_threads_arg; +#endif + // Figure out what cores to bind to + if (args.cores_given) { + const char **core_list = NULL; + int len = 0; + split_string(args.cores_arg, &len, &core_list); + zconf.pin_cores_len = (uint32_t)len; + zconf.pin_cores = + xcalloc(zconf.pin_cores_len, sizeof(uint32_t)); + for (uint32_t i = 0; i < zconf.pin_cores_len; ++i) { + zconf.pin_cores[i] = atoi(core_list[i]); + } + } else { + int num_cores = get_num_cores(); + zconf.pin_cores_len = (uint32_t)num_cores; + zconf.pin_cores = + xcalloc(zconf.pin_cores_len, sizeof(uint32_t)); + for (uint32_t i = 0; i < zconf.pin_cores_len; ++i) { + zconf.pin_cores[i] = i; + } + } + +// PFRING +#ifdef PFRING +#define MAX_CARD_SLOTS 32768 +#define QUEUE_LEN 8192 +#define ZMAP_PF_BUFFER_SIZE 1536 +#define ZMAP_PF_ZC_CLUSTER_ID 9627 + uint32_t user_buffers = zconf.senders * 256; + uint32_t queue_buffers = zconf.senders * QUEUE_LEN; + uint32_t card_buffers = 2 * MAX_CARD_SLOTS; + uint32_t total_buffers = + user_buffers + queue_buffers + card_buffers + 2; + uint32_t metadata_len = 0; + uint32_t numa_node = 0; // TODO + zconf.pf.cluster = pfring_zc_create_cluster( + ZMAP_PF_ZC_CLUSTER_ID, ZMAP_PF_BUFFER_SIZE, metadata_len, + total_buffers, numa_node, NULL, NULL); + if (zconf.pf.cluster == NULL) { + log_fatal("zmap", "Could not create zc cluster: %s", + strerror(errno)); + } + + zconf.pf.buffers = xcalloc(user_buffers, sizeof(pfring_zc_pkt_buff *)); + for (uint32_t i = 0; i < user_buffers; ++i) { + zconf.pf.buffers[i] = + pfring_zc_get_packet_handle(zconf.pf.cluster); + if (zconf.pf.buffers[i] == NULL) { + log_fatal("zmap", "Could not get ZC packet handle"); + } + } + + zconf.pf.send = + pfring_zc_open_device(zconf.pf.cluster, zconf.iface, tx_only, 0); + if (zconf.pf.send == NULL) { + log_fatal("zmap", "Could not open device %s for TX. [%s]", + zconf.iface, strerror(errno)); + } + + zconf.pf.recv = + pfring_zc_open_device(zconf.pf.cluster, zconf.iface, rx_only, 0); + if (zconf.pf.recv == NULL) { + log_fatal("zmap", "Could not open device %s for RX. [%s]", + zconf.iface, strerror(errno)); + } + + zconf.pf.queues = xcalloc(zconf.senders, sizeof(pfring_zc_queue *)); + for (uint32_t i = 0; i < zconf.senders; ++i) { + zconf.pf.queues[i] = + pfring_zc_create_queue(zconf.pf.cluster, QUEUE_LEN); + if (zconf.pf.queues[i] == NULL) { + log_fatal("zmap", "Could not create queue: %s", + strerror(errno)); + } + } + + zconf.pf.prefetches = pfring_zc_create_buffer_pool(zconf.pf.cluster, 8); + if (zconf.pf.prefetches == NULL) { + log_fatal("zmap", "Could not open prefetch pool: %s", + strerror(errno)); + } +#endif + + // resume scan if requested + + start_zmap(); + + fclose(log_location); + + cmdline_parser_free(&args); + free(params); + return EXIT_SUCCESS; +} diff --git a/src/zmap_schema.py b/src/zmap_schema.py new file mode 100644 index 0000000..05e8423 --- /dev/null +++ b/src/zmap_schema.py @@ -0,0 +1,93 @@ +from zschema.leaves import * +from zschema.compounds import * +import zschema.registry + +zmap_base = Record({ + "saddr":IPv4Address(), + "saddr_raw":Unsigned32BitInteger(), + "daddr":IPv4Address(), + "daddr_raw":Unsigned32BitInteger(), + "ipid":Unsigned32BitInteger(), + "ttl":Unsigned32BitInteger(), + "classification":String(), + "success":Unsigned32BitInteger(), + "app_success":Unsigned32BitInteger(), + "repeat":Unsigned32BitInteger(), + "cooldown":Unsigned32BitInteger(), + "timestamp_str":String(), + "timestamp_ts":Unsigned32BitInteger(), + "timestamp_us":Unsigned32BitInteger(), + "icmp_responder":String(), + "icmp_type":Unsigned32BitInteger(), + "icmp_code":Unsigned32BitInteger(), + "icmp_unreach_str":String(), + "sport":Unsigned32BitInteger(), + "dport":Unsigned32BitInteger(), + "data":String(), + "length":Unsigned32BitInteger(), + +}) + +zmap_upnp = Record({ + "type":String(), + "server":AnalyzedString(), + "location":AnalyzedString(), + "usn":String(), + "st":String(), + "ext":String(), + "cache_control":String(), + "x_user_agent":String(), + "agent":String(), + "date":String(), +}, extends=zmap_base) + +zschema.registry.register_schema("zmap-upnp", zmap_upnp) + + +dns_question = SubRecord({ + "name":String(), + "qtype":Unsigned32BitInteger(), + "qtype_str":String(), + "qclass":Unsigned32BitInteger(), +}) + +dns_answer = SubRecord({ + "name":String(), + "type":Unsigned32BitInteger(), + "type_str":String(), + "class":Unsigned32BitInteger(), + "ttl":Unsigned32BitInteger(), + "rdlength":Unsigned32BitInteger(), + "rdata_is_parsed":Unsigned32BitInteger(), + "rdata":String(), # hex +}) + +zmap_dns = Record({ + "qr":Unsigned16BitInteger(), + "rcode":Unsigned16BitInteger(), + "dns_id":Unsigned32BitInteger(), + "dns_rd":Unsigned32BitInteger(), + "dns_tc":Unsigned32BitInteger(), + "dns_aa":Unsigned32BitInteger(), + "dns_opcode":Unsigned32BitInteger(), + "dns_qr":Unsigned32BitInteger(), + "dns_rcode":Unsigned32BitInteger(), + "dns_cd":Unsigned32BitInteger(), + "dns_ad":Unsigned32BitInteger(), + "dns_z":Unsigned32BitInteger(), + "dns_ra":Unsigned32BitInteger(), + "dns_qdcount":Unsigned32BitInteger(), + "dns_ancount":Unsigned32BitInteger(), + "dns_nscount":Unsigned32BitInteger(), + "dns_arcount":Unsigned32BitInteger(), + "dns_questions":ListOf(dns_question), + "dns_answers":ListOf(dns_answer), + "dns_authorities":ListOf(dns_answer), + "dns_additionals":ListOf(dns_answer), + "dns_unconsumed_bytes":Unsigned32BitInteger(), + "dns_parse_err":Unsigned32BitInteger(), + "raw_data":String(), + "udp_len":Unsigned32BitInteger(), +}, extends=zmap_base) + +zschema.registry.register_schema("zmap-dns", zmap_dns) diff --git a/src/zopt.ggo.in b/src/zopt.ggo.in new file mode 100644 index 0000000..5346ec2 --- /dev/null +++ b/src/zopt.ggo.in @@ -0,0 +1,210 @@ +# ZMap Copyright 2013 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# zmap option description to be processed by gengetopt + +package "zmap" +version "@ZMAP_VERSION@" +purpose "A fast Internet-wide scanner." + +section "Basic Arguments" + +option "target-ports" p "comma-delimited list of ports and port ranges to scan (for TCP and UDP scans)" + typestr="ports" + optional string +option "output-file" o "Output file" + typestr="name" + optional string +option "blocklist-file" b "File of subnets to exclude, in CIDR notation, e.g. 192.168.0.0/16" + typestr="path" + optional string +option "allowlist-file" w "File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16" + typestr="path" + optional string +option "list-of-ips-file" I "List of individual addresses to scan in random order. Use --allowlist-file unless >1 million IPs" + typestr="path" + optional string + + +section "Scan Options" + +option "rate" r "Set send rate in packets/sec" + typestr="pps" + optional int +option "bandwidth" B "Set send rate in bits/second (supports suffixes G, M and K)" + typestr="bps" + optional string +option "batch" - "Set batch size for how many packets to send in a single syscall. Only advantageous on Linux (default=64)" + typestr="pps" + optional int +option "max-targets" n "Cap number of targets to probe (as a number or a percentage of the address space)" + typestr="n" + optional string +option "max-runtime" t "Cap length of time for sending packets" + typestr="secs" + optional int +option "max-results" N "Cap number of results to return" + typestr="n" + optional int +option "probes" P "Number of probes to send to each IP/Port pair" + typestr="n" + default="1" + optional int +option "cooldown-time" c "How long to continue receiving after sending last probe" + typestr="secs" + default="8" + optional int +option "seed" e "Seed used to select address permutation" + typestr="n" + optional longlong +option "retries" - "Max number of times to try to send packet if send fails" + typestr="n" + default="10" + optional int +option "dryrun" d "Don't actually send packets" + optional + + +section "Scan Sharding" + +option "shards" - "Set the total number of shards" + typestr="N" + optional int + default="1" +option "shard" - "Set which shard this scan is (0 indexed)" + typestr="n" + optional int + default="0" + +section "Network Options" + +option "source-port" s "Source port(s) for scan packets" + typestr="port|range" + optional string +option "source-ip" S "Source address(es) for scan packets" + typestr="ip|range" + optional string +option "gateway-mac" G "Specify gateway MAC address" + typestr="addr" + optional string +option "source-mac" - "Source MAC address" + typestr="addr" + optional string +option "interface" i "Specify network interface to use" + typestr="name" + optional string +option "iplayer" X "Sends IP packets instead of Ethernet (for VPNs)" + optional + +section "Probe Modules" +option "probe-module" M "Select probe module" + typestr="name" + default="tcp_synscan" + optional string +option "probe-args" - "Arguments to pass to probe module" + typestr="args" + optional string +option "probe-ttl" - "Set TTL value for probe IP packets" + typestr="n" + default="255" + optional int +option "list-probe-modules" - "List available probe modules" + optional + +section "Results Output" +option "output-fields" f "Fields that should be output in result set" + typestr="fields" + optional string +option "output-module" O "Select output module" + typestr="name" + optional string +option "output-args" - "Arguments to pass to output module" + typestr="args" + optional string +option "output-filter" - "Specify a filter over the response fields to limit what responses get sent to the output module" + typestr="filter" + optional string +option "list-output-modules" - "List available output modules" + optional +option "list-output-fields" - "List all fields that can be output by selected probe module" + optional +option "no-header-row" - "Precludes outputting any header rows in data (e.g., CSV headers)" + optional + + +section "Response Deduplication" +option "dedup-method" - "Specifies how response deduplication should be performed. Options: default, none, full, window" + typestr="method" + optional string +option "dedup-window-size" - "Specifies window size for how many recent responses to keep in memory for deuplication" + typestr="targets" + default="1000000" + optional int + +section "Logging and Metadata" +option "verbosity" v "Level of log detail (0-5)" + typestr="n" + default="3" + optional int +option "log-file" l "Write log entries to file" + typestr="name" + optional string +option "log-directory" L "Write log entries to a timestamped file in this directory" + typestr="directory" + optional string +option "metadata-file" m "Output file for scan metadata (JSON)" + typestr="name" + optional string +option "status-updates-file" u "Write scan progress updates to CSV file" + typestr="name" + optional string +option "quiet" q "Do not print status updates" + optional +option "disable-syslog" - "Disables logging messages to syslog" + optional +option "notes" - "Inject user-specified notes into scan metadata" + typestr="notes" + optional string +option "user-metadata" - "Inject user-specified JSON metadata into scan metadata" + typestr="json" + optional string + +section "Additional Options" +option "config" C "Read a configuration file, which can specify any of these options" + typestr="filename" + default="/etc/zmap/zmap.conf" + optional string + +option "max-sendto-failures" - "Maximum NIC sendto failures before scan is aborted" + typestr="n" + default="-1" + optional int + +option "min-hitrate" - "Minimum hitrate that scan can hit before scan is aborted" + typestr="n" + default="0.0" + optional float + +option "sender-threads" T "Threads used to send packets" + typestr="n" + default="4" + optional int + +option "cores" - "Comma-separated list of cores to pin to" + optional string +option "ignore-blocklist-errors" - "Ignore invalid entries in allowlist/blocklist file." + optional +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +text "\nExamples:\n\ + zmap -p 80 (scan full IPv4 address space for hosts on TCP/80)\n\ + zmap -N 5 -B 10M -p 80 (find 5 HTTP servers, scanning at 10 Mb/s)\n\ + zmap -p 80 10.0.0.0/8 192.168.0.0/16 (scan both subnets on TCP/80)\n\ + zmap -p 80 1.2.3.4 10.0.0.3 (scan 1.2.3.4, 10.0.0.3 on TCP/80)\n\ + zmap -p 80,100-102 (scan full IPv4 on ports 80, 100, 101, 102)" diff --git a/src/zopt_compat.c b/src/zopt_compat.c new file mode 100644 index 0000000..f5f406d --- /dev/null +++ b/src/zopt_compat.c @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "zopt.c" diff --git a/src/ztee.1 b/src/ztee.1 new file mode 100644 index 0000000..036063b --- /dev/null +++ b/src/ztee.1 @@ -0,0 +1,57 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "ZTEE" "1" "August 2021" "ZMap" "ztee" +. +.SH "NAME" +\fBztee\fR \- output buffer and splitter +. +.SH "SYNOPSIS" +ztee [ OPTIONS\.\.\. ] [ FILE\.\.\. ] +. +.SH "DESCRIPTION" +\fIZTee\fR is an output buffer and splitter for use with ZMap output data\. ZTee should be used whenever ZMap is piped into an application scanner, placed between ZMap and the application scanner\. ZTee writes the transformed output to stdout, and writes the original output to FILE\. +. +.P +See \fB\-\-help\fR for examples\. +. +.SH "CSV PROCESSING AND RAW MODE" +\fIZTee\fR operates by default on CSV\-format output from ZMap\. It only outputs IP addresses (from the input\'s \fBip\fR or \fBsaddr\fR field) to stdout, while writing all input to the output file\. ZTee does not print the first line of input to stdout, since that row is the CSV header\. +. +.P +To operate on data in any other format, pass the \fB\-\-raw\fR flag\. In raw mode, ztee behaves like tee: it will not transform or attempt to parse the input data\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fB\-r\fR, \fB\-\-raw\fR +Ignore input formatting and pass through raw input\. This causes ztee to behave exactly like tee, with the addition of buffering\. +. +.TP +\fB\-\-success\-only\fR +Only write to stdout rows where success=1 or success=true\. Invalid in combination with \fB\-\-raw\fR\. +. +.TP +\fB\-m\fR, \fB\-\-monitor\fR +Print monitor data to stderr +. +.TP +\fB\-u\fR, \fB\-\-status\-updates\-file\fR +Write status updates (monitor data) to the given file, in CSV format +. +.TP +\fB\-l\fR, \fB\-\-log\-file=name\fR +Write errors etc\. to the given file\. If none, ZTee logs to stderr\. +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-h, \-\-help\fR +Display help +. +.TP +\fB\-V, \-\-version\fR +Display version + diff --git a/src/ztee.1.html b/src/ztee.1.html new file mode 100644 index 0000000..a446707 --- /dev/null +++ b/src/ztee.1.html @@ -0,0 +1,129 @@ + + + + + + ztee(1) - output buffer and splitter + + + + +
+ + + +
    +
  1. ztee(1)
    2. +
  2. ztee
    3. +
  3. ztee(1)
    4. +
+ +

NAME

+

+ ztee - output buffer and splitter +

+ +

SYNOPSIS

+ +

ztee [ OPTIONS... ] [ FILE... ]

+ +

DESCRIPTION

+ +

ZTee is an output buffer and splitter for use with ZMap output data. ZTee +should be used whenever ZMap is piped into an application scanner, placed +between ZMap and the application scanner. ZTee writes the transformed output +to stdout, and writes the original output to FILE.

+ +

See --help for examples.

+ +

CSV PROCESSING AND RAW MODE

+ +

ZTee operates by default on CSV-format output from ZMap. It only outputs IP +addresses (from the input's ip or saddr field) to stdout, while writing all +input to the output file. ZTee does not print the first line of input to stdout, +since that row is the CSV header.

+ +

To operate on data in any other format, pass the --raw flag. In raw mode, ztee +behaves like tee: it will not transform or attempt to parse the input data.

+ +

OPTIONS

+ +

BASIC OPTIONS

+ +
+
-r, --raw

Ignore input formatting and pass through raw input. This causes +ztee to behave exactly like tee, with the addition of buffering.

+
--success-only

Only write to stdout rows where success=1 or success=true. Invalid +in combination with --raw.

+
-m, --monitor

Print monitor data to stderr

+
-u, --status-updates-file

Write status updates (monitor data) to the given file, in CSV format

+
-l, --log-file=name

Write errors etc. to the given file. If none, ZTee logs to stderr.

+
+ + +

ADDITIONAL OPTIONS

+ +
+
-h, --help

Display help

+
-V, --version

Display version

+
+ + + +
    +
  1. ZMap
    2. +
  2. August 2021
    3. +
  3. ztee(1)
    4. +
+ +
+ + diff --git a/src/ztee.1.ronn b/src/ztee.1.ronn new file mode 100644 index 0000000..8d9ef12 --- /dev/null +++ b/src/ztee.1.ronn @@ -0,0 +1,54 @@ +ztee(1) - output buffer and splitter +==================================== + +## SYNOPSIS + +ztee [ OPTIONS... ] [ FILE... ] + +## DESCRIPTION + +*ZTee* is an output buffer and splitter for use with ZMap output data. ZTee +should be used whenever ZMap is piped into an application scanner, placed +between ZMap and the application scanner. ZTee writes the transformed output +to stdout, and writes the original output to FILE. + +See `--help` for examples. + +## CSV PROCESSING AND RAW MODE + +*ZTee* operates by default on CSV-format output from ZMap. It only outputs IP +addresses (from the input's `ip` or `saddr` field) to stdout, while writing all +input to the output file. ZTee does not print the first line of input to stdout, +since that row is the CSV header. + +To operate on data in any other format, pass the `--raw` flag. In raw mode, ztee +behaves like tee: it will not transform or attempt to parse the input data. + +## OPTIONS + +### BASIC OPTIONS ### + + * `-r`, `--raw`: + Ignore input formatting and pass through raw input. This causes + ztee to behave exactly like tee, with the addition of buffering. + + * `--success-only`: + Only write to stdout rows where success=1 or success=true. Invalid + in combination with `--raw`. + + * `-m`, `--monitor`: + Print monitor data to stderr + + * `-u`, `--status-updates-file`: + Write status updates (monitor data) to the given file, in CSV format + + * `-l`, `--log-file=name`: + Write errors etc. to the given file. If none, ZTee logs to stderr. + +### ADDITIONAL OPTIONS ### + + * `-h, --help`: + Display help + + * `-V, --version`: + Display version diff --git a/src/ztee.c b/src/ztee.c new file mode 100644 index 0000000..29b610c --- /dev/null +++ b/src/ztee.c @@ -0,0 +1,555 @@ +/* + * ZTee Copyright 2014 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// without defining this, FreeBSD throws a warning. +#define _WITH_GETLINE +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../lib/lockfd.h" +#include "../lib/logger.h" +#include "../lib/queue.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" +#include "../lib/csv.h" + +#include "topt.h" + +typedef enum file_format { FORMAT_CSV, FORMAT_JSON, FORMAT_RAW } format_t; +static const char *format_names[] = {"csv", "json", "raw"}; + +typedef struct ztee_conf { + // Files + char *output_filename; + char *status_updates_filename; + char *log_file_name; + FILE *output_file; + FILE *status_updates_file; + FILE *log_file; + + // Log level + int log_level; + + // Input formats + format_t in_format; + format_t out_format; + + // Output config + int success_only; + + // Monitor config + int monitor; + + // Field indices + size_t ip_field; + size_t success_field; + +} ztee_conf_t; + +static ztee_conf_t tconf; + +static int print_from_csv(char *line); + +static format_t test_input_format(char *line, size_t len) +{ + // Check for empty input, remember line contains '\n' + if (len < 2) { + return FORMAT_RAW; + } + if (len >= 3) { + // If the input is JSON, the line should look like + // {.......}\n + if (line[0] == '{' && line[len - 2] == '}') { + return FORMAT_JSON; + } + } + if (strchr(line, ',') != NULL) { + return FORMAT_CSV; + } + return FORMAT_RAW; +} + +int done = 0; +int process_done = 0; +int total_read_in = 0; +int read_in_last_sec = 0; +int total_written = 0; + +double start_time; + +pthread_t threads[3]; + +// one thread reads in +// one thread writes out and parses + +// pops next element and determines what to do +// if zqueue_t is empty and read_in is finished, then +// it exits +void *process_queue(void *my_q); + +// uses fgets to read from stdin and add it to the zqueue_t +void *read_in(void *my_q); + +// does the same as find UP but finds only successful IPs, determined by the +// is_successful field and flag +void find_successful_IP(char *my_string); + +// finds IP in the string of csv and sends it to stdout for zgrab +// you need to know what position is the csv string the ip field is in +// zero indexed +void find_IP(char *my_string); + +// writes a csv string out to csv file +// fprintf(stderr, "Is empty inside if %i\n", is_empty(queue)); +void write_out_to_file(char *data); + +// figure out how many fields are present if it is a csv +void figure_out_fields(char *data); + +// check that the output file is either in a csv form or json form +// throws error is it is not either +// NOTE: JSON OUTPUT NOT IMPLEMENTED +void output_file_is_csv(void); + +void print_thread_error(char *string); + +// monitor code for ztee +// executes every second +void *monitor_ztee(void *my_q); + +#define SET_IF_GIVEN(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = args.ARG##_arg; \ + }; \ + } +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char *argv[]) +{ + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + assert(params); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + signal(SIGPIPE, SIG_IGN); + + // Handle help text and version + if (args.help_given) { + cmdline_parser_print_help(); + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + // Try opening the log file + tconf.log_level = ZLOG_WARN; + if (args.log_file_given) { + tconf.log_file = fopen(args.log_file_arg, "w"); + } else { + tconf.log_file = stderr; + } + + // Check for an error opening the log file + if (tconf.log_file == NULL) { + log_init(stderr, tconf.log_level, 0, "ztee"); + log_fatal("ztee", "Could not open log file"); + } + + // Actually init the logging infrastructure + log_init(tconf.log_file, tconf.log_level, 0, "ztee"); + + // Check for an output file + if (args.inputs_num < 1) { + log_fatal("ztee", "No output file specified"); + } + if (args.inputs_num > 1) { + log_fatal("ztee", "Extra positional arguments starting with %s", + args.inputs[1]); + } + + tconf.output_filename = args.inputs[0]; + tconf.output_file = fopen(tconf.output_filename, "w"); + if (!tconf.output_file) { + log_fatal("ztee", "Could not open output file %s, %s", + tconf.output_filename, strerror(errno)); + } + + // Read actual options + int raw = 0; + SET_BOOL(tconf.success_only, success_only); + SET_BOOL(tconf.monitor, monitor); + SET_BOOL(raw, raw); + + // Open the status update file if necessary + if (args.status_updates_file_given) { + // Try to open the status output file + char *filename = args.status_updates_file_arg; + FILE *file = fopen(filename, "w"); + if (!file) { + char *err = strerror(errno); + log_fatal("ztee", + "unable to open status updates file %s (%s)", + filename, err); + } + // Set the variables in state + tconf.status_updates_filename = filename; + tconf.status_updates_file = file; + } + + // Read the first line of the input file + size_t first_line_len = 1024; + char *first_line = xmalloc(first_line_len); + if (getline(&first_line, &first_line_len, stdin) < 0) { + log_fatal("ztee", "reading input to test format failed"); + } + // Detect the input format + if (!raw) { + format_t format = test_input_format(first_line, first_line_len); + log_info("ztee", "detected input format %s", + format_names[format]); + tconf.in_format = format; + } else { + tconf.in_format = FORMAT_RAW; + log_info("ztee", "raw input"); + } + + if (tconf.in_format == FORMAT_JSON) { + log_fatal("ztee", "json input not implemented"); + } + + // Find fields if needed + char *header = strdup(first_line); + int found_success = 0; + int found_ip = 0; + if (tconf.in_format == FORMAT_CSV) { + static const char *success_names[] = {"success"}; + static const char *ip_names[] = {"saddr", "ip"}; + int success_idx = csv_find_index(header, success_names, 1); + if (success_idx >= 0) { + found_success = 1; + tconf.success_field = (size_t)success_idx; + } + int ip_idx = csv_find_index(header, ip_names, 2); + if (found_ip >= 0) { + found_ip = 1; + tconf.ip_field = (size_t)ip_idx; + } + if (!found_ip) { + log_fatal("ztee", "Unable to find IP/SADDR field"); + } + } + + if (tconf.success_only) { + if (tconf.in_format != FORMAT_CSV) { + log_fatal("ztee", "success filter requires csv input"); + } + if (!found_success) { + log_fatal("ztee", "Could not find success field"); + } + } + + // Make the queue + zqueue_t *queue = queue_init(); + assert(queue); + + // Add the first line to the queue if needed + push_back(first_line, queue); + + // Start the regular read thread + pthread_t read_thread; + if (pthread_create(&read_thread, NULL, read_in, queue)) { + log_fatal("ztee", "unable to start read thread"); + } + + // Record the start time + start_time = now(); + + // Start the process thread + pthread_t process_thread; + if (pthread_create(&process_thread, NULL, process_queue, queue)) { + log_fatal("ztee", "unable to start process thread"); + } + + // Start the monitor thread if necessary, and join to it + if (tconf.monitor || tconf.status_updates_file) { + pthread_t monitor_thread; + if (pthread_create(&monitor_thread, NULL, monitor_ztee, + queue)) { + log_fatal("ztee", "unable to create monitor thread"); + } + pthread_join(monitor_thread, NULL); + } + + // Join to the remaining threads, + pthread_join(read_thread, NULL); + pthread_join(process_thread, NULL); + return 0; +} + +void *process_queue(void *arg) +{ + zqueue_t *queue = arg; + FILE *output_file = tconf.output_file; + while (!process_done) { + + pthread_mutex_lock(&queue->lock); + while (!done && is_empty(queue)) { + pthread_cond_wait(&queue->empty, &queue->lock); + } + if (done && is_empty(queue)) { + process_done = 1; + pthread_mutex_unlock(&queue->lock); + continue; + } + znode_t *node = pop_front_unsafe(queue); + pthread_mutex_unlock(&queue->lock); + + // Write raw data to output file + fprintf(output_file, "%s", node->data); + fflush(output_file); + if (ferror(output_file)) { + log_fatal("ztee", "Error writing to output file"); + } + + // Dump to stdout + switch (tconf.in_format) { + case FORMAT_JSON: + log_fatal("ztee", "JSON input format unimplemented"); + break; + case FORMAT_CSV: + print_from_csv(node->data); + break; + default: + // Handle raw + fprintf(stdout, "%s", node->data); + break; + } + + // Check to see if write failed + fflush(stdout); + if (ferror(stdout)) { + log_fatal("ztee", "%s", "Error writing to stdout"); + } + + // Record output lines + total_written++; + + // Free the memory + free(node->data); + free(node); + } + process_done = 1; + fflush(output_file); + fclose(output_file); + return NULL; +} + +void *read_in(void *arg) +{ + // Allocate buffers + zqueue_t *queue = (zqueue_t *)arg; + size_t length = 1000; + char *input = xcalloc(sizeof(char), length); + ; + + // Read in from stdin and add to back of linked list + while (getline(&input, &length, stdin) > 0) { + push_back(input, queue); + + total_read_in++; + read_in_last_sec++; + } + pthread_mutex_lock(&queue->lock); + done = 1; + pthread_cond_signal(&queue->empty); + pthread_mutex_unlock(&queue->lock); + return NULL; +} + +int print_from_csv(char *line) +{ + if (total_written == 0) { + return 1; + } + if (tconf.success_only) { + char *success_entry = csv_get_index(line, tconf.success_field); + if (success_entry == NULL) { + return 1; + } + int success = 0; + if (atoi(success_entry)) { + success = 1; + } else if (strcasecmp(success_entry, "true") == 0) { + success = 1; + } + if (!success) { + return 1; + } + } + // Find the ip + char *ip = csv_get_index(line, tconf.ip_field); + int ret = fprintf(stdout, "%s\n", ip); + if (ferror(stdout)) { + log_fatal("ztee", "unable to write to stdout"); + } + return ret; +} + +void output_file_is_csv(void) +{ + return; + /* + char *dot = strrchr(output_filename); + if dot == NULL { + return; + } + */ + /* + int length = strlen(output_filename); + char *end_of_file = malloc(sizeof(char*) *4); + strncpy(end_of_file, output_filename+(length - 3), 3); + end_of_file[4] = '\0'; + const char *csv = "csv\n"; + const char *json = "jso\n"; + if(!strncmp(end_of_file, csv, 3) && !strncmp(end_of_file, json, 3)){ + log_fatal("ztee", "Invalid output format"); + } + if(!strncmp(end_of_file, csv, 3)) output_csv = 1; + if(!strncmp(end_of_file, json, 3)) output_csv = 0; + */ +} + +void print_thread_error(char *string) +{ + fprintf(stderr, "Could not create thread %s\n", string); + return; +} + +#define TIME_STR_LEN 20 + +typedef struct ztee_stats { + // Read stats + uint32_t total_read; + uint32_t read_per_sec_avg; + uint32_t read_last_sec; + + // Buffer stats + uint32_t buffer_cur_size; + uint32_t buffer_avg_size; + uint64_t _buffer_size_sum; + + // Duration + double _last_age; + uint32_t time_past; + char time_past_str[TIME_STR_LEN]; +} stats_t; + +void update_stats(stats_t *stats, zqueue_t *queue) +{ + double age = now() - start_time; + double delta = age - stats->_last_age; + stats->_last_age = age; + + stats->time_past = age; + time_string((int)age, 0, stats->time_past_str, TIME_STR_LEN); + + uint32_t total_read = total_read_in; + stats->read_last_sec = (total_read - stats->total_read) / delta; + stats->total_read = total_read; + stats->read_per_sec_avg = stats->total_read / age; + + stats->buffer_cur_size = get_size(queue); + stats->_buffer_size_sum += stats->buffer_cur_size; + stats->buffer_avg_size = stats->_buffer_size_sum / age; +} + +void *monitor_ztee(void *arg) +{ + zqueue_t *queue = (zqueue_t *)arg; + stats_t *stats = xmalloc(sizeof(stats_t)); + + if (tconf.status_updates_file) { + fprintf( + tconf.status_updates_file, + "time_past,total_read_in,read_in_last_sec,read_per_sec_avg," + "buffer_current_size,buffer_avg_size\n"); + fflush(tconf.status_updates_file); + if (ferror(tconf.status_updates_file)) { + log_fatal("ztee", + "unable to write to status updates file"); + } + } + while (!process_done) { + sleep(1); + + update_stats(stats, queue); + if (tconf.monitor) { + lock_file(stderr); + fprintf( + stderr, + "%5s read_rate: %u rows/s (avg %u rows/s), buffer_size: %u (avg %u)\n", + stats->time_past_str, stats->read_last_sec, + stats->read_per_sec_avg, stats->buffer_cur_size, + stats->buffer_avg_size); + fflush(stderr); + unlock_file(stderr); + if (ferror(stderr)) { + log_fatal( + "ztee", + "unable to write status updates to stderr"); + } + } + if (tconf.status_updates_file) { + fprintf(tconf.status_updates_file, + "%u,%u,%u,%u,%u,%u\n", stats->time_past, + stats->total_read, stats->read_last_sec, + stats->read_per_sec_avg, stats->buffer_cur_size, + stats->buffer_avg_size); + fflush(tconf.status_updates_file); + if (ferror(tconf.status_updates_file)) { + log_fatal( + "ztee", + "unable to write to status updates file"); + } + } + } + if (tconf.monitor) { + lock_file(stderr); + fflush(stderr); + unlock_file(stderr); + } + if (tconf.status_updates_file) { + fflush(tconf.status_updates_file); + fclose(tconf.status_updates_file); + } + return NULL; +} diff --git a/src/ztopt.ggo.in b/src/ztopt.ggo.in new file mode 100644 index 0000000..d1c565d --- /dev/null +++ b/src/ztopt.ggo.in @@ -0,0 +1,18 @@ +# ZTests Copyright 2014 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# ztests option description to be processed by gengetopt + +package "ztests" +version "@ZMAP_VERSION@" +purpose "A tests harness tool for zmap" + +section "Additional options" + +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional diff --git a/src/ztopt_compat.c b/src/ztopt_compat.c new file mode 100644 index 0000000..e7b7b8c --- /dev/null +++ b/src/ztopt_compat.c @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "ztopt.c" diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..e705b42 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,6 @@ +*.*-t* +*-t* +tempfile +outfile +!.gitignore +shardfile diff --git a/test/configs/blocklist_shard.conf b/test/configs/blocklist_shard.conf new file mode 100644 index 0000000..2ad400e --- /dev/null +++ b/test/configs/blocklist_shard.conf @@ -0,0 +1,25 @@ +# From IANA IPv4 Special-Purpose Address Registry +# http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +# Updated 2013-05-22 + +0.0.0.0/8 # RFC1122: "This host on this network" +10.0.0.0/8 # RFC1918: Private-Use +100.64.0.0/10 # RFC6598: Shared Address Space +127.0.0.0/8 # RFC1122: Loopback +169.254.0.0/16 # RFC3927: Link Local +172.16.0.0/12 # RFC1918: Private-Use +192.0.0.0/24 # RFC6890: IETF Protocol Assignments +192.0.2.0/24 # RFC5737: Documentation (TEST-NET-1) +192.88.99.0/24 # RFC3068: 6to4 Relay Anycast +#192.168.0.0/16 # RFC1918: Private-Use +198.18.0.0/15 # RFC2544: Benchmarking +198.51.100.0/24 # RFC5737: Documentation (TEST-NET-2) +203.0.113.0/24 # RFC5737: Documentation (TEST-NET-3) +240.0.0.0/4 # RFC1112: Reserved +255.255.255.255/32 # RFC0919: Limited Broadcast + +# From IANA Multicast Address Space Registry +# http://www.iana.org/assignments/multicast-addresses/multicast-addresses.xhtml +# Updated 2013-06-25 + +224.0.0.0/4 # RFC5771: Multicast/Reserved diff --git a/test/test-shard.sh b/test/test-shard.sh new file mode 100755 index 0000000..50d89f5 --- /dev/null +++ b/test/test-shard.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +SHARDS=19 +for i in {0..18}; do + ../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 1234 --shards=$SHARDS --shard=$i 1.2.3.0/24 | grep daddr | awk '{print $7}' > s$i.scanned + ../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 8675309 --shards=$SHARDS --shard=$i 1.2.3.0/24 -T 3 | grep daddr | awk '{print $7}' > t$i.scanned +done + +SHARDS=258 +for i in {0..257}; do + ../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 11043 --shards=$SHARDS --shard=$i 1.2.0.0/16 | grep daddr | awk '{print $7}' > bs$i.scanned + ../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 829 --shards=$SHARDS --shard=$i 1.2.0.0/16 -T 3 | grep daddr | awk '{print $7}' > bt$i.scanned +done + + +../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 18172017 1.2.3.0/24 -T 3 | grep daddr | awk '{print $7}' > one_t3$i.scanned +../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 13332727813 1.2.3.0/24 -T 1 | grep daddr | awk '{print $7}' > one_one$i.scanned + +echo "19 Shards - Expect: 256" +cat s*.scanned | wc -l +echo "Duplicates:" +cat s*.scanned | sort | uniq -d | wc -l +echo "" + +echo "258 Shards - Expect: 65536" +cat bs*.scanned | wc -l +echo "Duplicates:" +cat bs*.scanned | sort | uniq -d | wc -l +echo "" + +echo "258 Shards, 3 Threads - Expect: 65536" +cat bt*.scanned | wc -l +echo "Duplicates:" +cat bt*.scanned | sort | uniq -d | wc -l +echo "" + + + +echo "19 Shards, 3 Threads - Expect: 256" +cat t*.scanned | wc -l +echo "Duplicates:" +cat t*.scanned | sort | uniq -d | wc -l +echo "" + +echo "1 Shard, 3 Threads - Expect: 256" +cat one_t*.scanned | wc -l +echo "Duplicates:" +cat one_t*.scanned | sort | uniq -d | wc -l +echo "" + +echo "1 Shard, 1 Thread - Expect: 256" +cat one_one*.scanned | wc -l +echo "Duplicates:" +cat one_one*.scanned | sort | uniq -d | wc -l +echo "" + diff --git a/test/test_big_group.sh b/test/test_big_group.sh new file mode 100755 index 0000000..1bf96fd --- /dev/null +++ b/test/test_big_group.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=0 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort > shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=1 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=2 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=3 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=4 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile + +cat shardfile | sort > temp +mv temp shardfile +echo "Line Count: (Should be 65536)" +cat shardfile | wc -l +echo "Duplicate Count" +cat shardfile | uniq -d | wc -l + +rm outfile +rm shardfile + +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=0 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort > shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=1 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=2 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=3 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=4 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile + +cat shardfile | sort > temp +mv temp shardfile +echo "Line Count: (Should be 196608)" +cat shardfile | wc -l +echo "Duplicate Count" +cat shardfile | uniq -d | wc -l + +rm outfile +rm shardfile diff --git a/test/test_sharding.py b/test/test_sharding.py new file mode 100644 index 0000000..4a13c87 --- /dev/null +++ b/test/test_sharding.py @@ -0,0 +1,74 @@ +import sh +import unittest + +from sh import cut, grep, cat, wc, uniq, mv + +zmap_std_args = [ "-b", + "configs/blocklist_shard.conf", + "--seed=1234", + "192.168.1.0/24", + "--dryrun", + "-c", + "1" + ] + +zmap = sh.Command("../src/zmap").bake(*zmap_std_args) + +def shard_file_name(shards, threads): + # Use naming conversion -t + return ''.join([str(shards), '-t', str(threads)]) + +def output_file_name(shards, shard, threads): + # Use naming convention: .-t + return ''.join([str(shards), '.', str(shard), '-t', str(threads)]) + +def parse(filename, **kwargs): + # cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | cut -d '.' -f 4 | sort -n | wc -l + return sh.sort(cut(cut(cut(grep(cat(filename), "ip"), d="|", f=2), d=" ", f=3), d=".", f=4), "-n", _out=kwargs.get("_out")) + +class TestSharding(unittest.TestCase): + + NUM_IPS = 256 + + def setUp(self): + pass + + def takeDown(self): + pass + + def _runTest(self, shards, max_threads): + for threads in range(1, max_threads + 1): + for shard in range(0, shards): + with sh.sudo: + outfile = output_file_name(shards, shard, threads) + zmap(p=80, T=threads, shards=shards, shard=shard, _out="tempfile") + parse("tempfile", _out=outfile) + dup_lines = int(wc(uniq(cat(outfile), "-d"), "-l")) + self.assertEqual(dup_lines, 0) + shard_file = shard_file_name(shards, threads) + if shard == 0: + cat(outfile, _out=shard_file) + else: + cat(shard_file, outfile, _out="tempfile") + mv("tempfile", shard_file) + + for threads in range(1, max_threads + 1): + shard_file = shard_file_name(shards, threads) + num_lines = int(wc(cat(shard_file), "-l")) + self.assertEqual(num_lines, TestSharding.NUM_IPS) + dup_lines = int(wc(uniq(sh.sort(cat(shard_file), "-n"), "-d"), "-l")) + self.assertEqual(dup_lines, 0) + + def testOneShard(self): + # Test with one shard + self._runTest(1, 4) + + + def testTwoShards(self): + self._runTest(2, 4) + +if __name__ == '__main__': + unittest.main() + + + diff --git a/test/test_zblocklist.py b/test/test_zblocklist.py new file mode 100644 index 0000000..e8e71c9 --- /dev/null +++ b/test/test_zblocklist.py @@ -0,0 +1,138 @@ +import unittest +import subprocess +import os +import sys + +executable_path = None + +class ZBlocklistTest(unittest.TestCase): + + BLOCKLIST = [ + "10.0.0.0/8 # private subnet", + "192.168.0.0/16 # private subnet", + "128.255.0.0/16 # university of iowa", + "141.212.120.0/24 # halderman lab" + ] + + ALLOWLIST = [ + "141.212.0.0/16 # university of michigan", + ] + + IPS = [ + "61.193.80.24", + "195.19.1.6", + "114.34.253.25", + "180.69.174.9", + "38.134.130.203", + "192.168.1.50", + "98.125.221.180", + "197.160.60.150", + "47.139.63.128", + "95.224.78.221", + "170.114.52.252", + "10.0.0.5", + "128.255.134.1", + "141.212.120.10", + "141.212.12.6" + ] + + IPS_MINUS_BL = [ + "61.193.80.24", + "195.19.1.6", + "114.34.253.25", + "180.69.174.9", + "38.134.130.203", + "98.125.221.180", + "197.160.60.150", + "47.139.63.128", + "95.224.78.221", + "170.114.52.252", + "141.212.12.6" + ] + + WL_IPS = [ + "141.212.120.10", + "141.212.12.6" + ] + + WL_IPS_MINUS_BL = [ + "141.212.12.6" + ] + + COMMENT_STRS = [ + "# some comment here", + " # some comment here", + ",google.com,data", + "\t#some comment here" + ] + + def setUp(self): + global executable_path + self.path = executable_path + with open("/tmp/blocklist", "w") as fd: + for line in self.BLOCKLIST: + fd.write("%s\n" % line) + with open("/tmp/allowlist", "w") as fd: + for line in self.ALLOWLIST: + fd.write("%s\n" % line) + with open("/tmp/ips", "w") as fd: + for line in self.IPS: + fd.write("%s\n" % line) + with open("/tmp/ips-commented", "w") as fd: + for line in self.IPS: + for comment in self.COMMENT_STRS: + fd.write("%s%s\n" % (line, comment)) + + def tearDown(self): + if os.path.exists("/tmp/blocklist"): + os.remove("/tmp/blocklist") + if os.path.exists("/tmp/allowlist"): + os.remove("/tmp/allowlist") + if os.path.exists("/tmp/ips"): + os.remove("/tmp/ips") + if os.path.exists("/tmp/ips-commented"): + os.remove("/tmp/ips-commented") + + + def execute(self, allowlist, blocklist, ipsfile="/tmp/ips", numtimestocat=1): + cmd = "cat" + for _ in range(0, numtimestocat): + cmd += " %s" % ipsfile + cmd += " | %s" % self.path + if allowlist: + cmd = cmd + " -w %s" % allowlist + if blocklist: + cmd = cmd + " -b %s" % blocklist + results = subprocess.check_output(cmd, shell=True) + ips = results.rstrip().split("\n") + return ips + + def testValidBlocklist(self): + res = self.execute(None, "/tmp/blocklist") + self.assertEqual(set(res), set(self.IPS_MINUS_BL)) + + def testValidAllowlist(self): + res = self.execute("/tmp/allowlist", None) + self.assertEqual(set(res), set(self.WL_IPS)) + + def testValidAllowAndBlockList(self): + res = self.execute("/tmp/allowlist", "/tmp/blocklist") + self.assertEqual(set(res), set(self.WL_IPS_MINUS_BL)) + + def testDuplicateChecking(self): + res = self.execute(None, "/tmp/blocklist", numtimestocat=5) + self.assertEqual(len(res), len(self.IPS_MINUS_BL)) + self.assertEqual(set(res), set(self.IPS_MINUS_BL)) + + def testCommentCharacters(self): + res = self.execute(None, "/tmp/blocklist", ipsfile="/tmp/ips-commented") + self.assertEqual(set(res), set(self.IPS_MINUS_BL)) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("USAGE: %s zblocklist" % sys.argv[0]) + sys.exit(1) + executable_path = sys.argv[1] + assert(os.path.exists(executable_path)) + unittest.main(argv=sys.argv[:1])