From 488b776ea5f50a627792d638305ca6e5851b2d28 Mon Sep 17 00:00:00 2001 From: Ductrung03 Date: Tue, 9 Jun 2026 10:27:04 +0700 Subject: [PATCH] feat: track DNS record first and last seen state --- dooked/include/utils/containers.hpp | 14 ++++-- dooked/include/utils/io_utils.hpp | 31 +++++++++++++ dooked/include/utils/probe_result.hpp | 12 +++++ dooked/source/cli_preprocessor.cpp | 65 +++++++++++++++++++++------ dooked/source/main.cpp | 6 +++ dooked/source/utils/io_utils.cpp | 16 +++++++ 6 files changed, 127 insertions(+), 17 deletions(-) diff --git a/dooked/include/utils/containers.hpp b/dooked/include/utils/containers.hpp index 7c955f6..bc9a614 100644 --- a/dooked/include/utils/containers.hpp +++ b/dooked/include/utils/containers.hpp @@ -45,9 +45,17 @@ template class map_container_t { void append_impl(std::string const &key, ValueType const &value) { auto &container = map_[key].dns_result_list_; - auto iter = std::find(container.cbegin(), container.cend(), value); - if (iter == container.cend()) { - container.push_back(value); + auto iter = std::find(container.begin(), container.end(), value); + if (iter != container.end()) { + iter->last_seen = ValueType::now_epoch(); + ++iter->seen_count; + } else { + auto rec = value; + auto const ts = ValueType::now_epoch(); + rec.first_seen = ts; + rec.last_seen = ts; + rec.seen_count = 1; + container.push_back(std::move(rec)); } } diff --git a/dooked/include/utils/io_utils.hpp b/dooked/include/utils/io_utils.hpp index 829b09e..e51d6bd 100644 --- a/dooked/include/utils/io_utils.hpp +++ b/dooked/include/utils/io_utils.hpp @@ -8,13 +8,29 @@ #include #include #include +#include +#include namespace dooked { using json = nlohmann::json; template using opt_list_t = std::optional>; +template +struct has_tracking_fields : std::false_type {}; + +template +struct has_tracking_fields().first_seen), + decltype(std::declval().last_seen), + decltype(std::declval().seen_count) +>> : std::true_type {}; + +template +inline constexpr bool has_tracking_fields_v = has_tracking_fields::value; + void to_json(json &j, probe_result_t const &record); +void from_json(json const &j, probe_result_t &record); dns_record_type_e dns_str_to_record_type(std::string const &); bool is_text_file(std::string const &file_extension); bool is_json_file(std::string const &file_extension); @@ -26,9 +42,12 @@ void trim(std::string &); struct json_data_t { std::string domain_name{}; std::string rdata{}; + std::uint64_t first_seen{}; + std::uint64_t last_seen{}; int ttl{}; int http_code{}; int content_length{}; + std::uint32_t seen_count{}; dns_record_type_e type{}; static json_data_t serialize(std::string const &d, int const len, @@ -42,6 +61,18 @@ struct json_data_t { data.ttl = json_object["ttl"].get(); data.content_length = len; data.http_code = http_code; + auto const it_fs = json_object.find("first-seen"); + if (it_fs != json_object.end() && it_fs->second.is_number()) { + data.first_seen = it_fs->second.get(); + } + auto const it_ls = json_object.find("last-seen"); + if (it_ls != json_object.end() && it_ls->second.is_number()) { + data.last_seen = it_ls->second.get(); + } + auto const it_sc = json_object.find("seen"); + if (it_sc != json_object.end() && it_sc->second.is_number()) { + data.seen_count = it_sc->second.get(); + } return data; } }; diff --git a/dooked/include/utils/probe_result.hpp b/dooked/include/utils/probe_result.hpp index 07211c6..4fbfb31 100644 --- a/dooked/include/utils/probe_result.hpp +++ b/dooked/include/utils/probe_result.hpp @@ -1,6 +1,8 @@ #pragma once #include "utils/constants.hpp" +#include +#include #include namespace dooked { @@ -12,6 +14,16 @@ struct probe_result_t { std::string rdata{}; dns_record_type_e type{}; // RR TYPE (2 octets) std::uint32_t ttl{}; // time to live(4 octets) + std::uint64_t first_seen{}; // epoch seconds, set on first observation + std::uint64_t last_seen{}; // epoch seconds, updated on every observation + std::uint32_t seen_count{}; // number of times this record was observed + + static std::uint64_t now_epoch() { + return static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); + } friend bool operator==(probe_result_t const &a, probe_result_t const &b) { return case_insensitive_compare(a.rdata, b.rdata) && (a.type == b.type); diff --git a/dooked/source/cli_preprocessor.cpp b/dooked/source/cli_preprocessor.cpp index c08d7fb..c908436 100644 --- a/dooked/source/cli_preprocessor.cpp +++ b/dooked/source/cli_preprocessor.cpp @@ -12,6 +12,8 @@ // defined (and assigned to) in main.cpp extern bool silent; extern bool compare_cl; +extern bool flag_first_seen; +extern bool flag_last_seen; namespace dooked { @@ -74,9 +76,12 @@ void compare_http_result(int const base_cl, json_data_t const &prev_http_result, case_insensitive_compare(a.rdata, b.rdata); }); if (!found) { - spdlog::error("[MISSING][{}][{}] `{}`", iter->domain_name, - dns_record_type_to_str(start_iter->type), - start_iter->rdata); + bool const any_filter = flag_first_seen || flag_last_seen; + if (!any_filter || flag_last_seen) { + spdlog::error("[MISSING][{}][{}] `{}`", iter->domain_name, + dns_record_type_to_str(start_iter->type), + start_iter->rdata); + } } } // information may have been changed @@ -98,9 +103,12 @@ void compare_http_result(int const base_cl, json_data_t const &prev_http_result, if (find_iter == eq_range.second) { auto const distance = std::distance(eq_range.first, eq_range.second); if (distance == 0) { - spdlog::error("[REMOVED][{}][{}] `{}`", iter->domain_name, - dns_record_type_to_str(start_iter->type), - start_iter->rdata); + bool const any_filter = flag_first_seen || flag_last_seen; + if (!any_filter || flag_last_seen) { + spdlog::error("[REMOVED][{}][{}] `{}`", iter->domain_name, + dns_record_type_to_str(start_iter->type), + start_iter->rdata); + } } else if (distance == 1) { spdlog::info("[CHANGED][{}][{}] from `{}` to `{}`", iter->domain_name, dns_record_type_to_str(start_iter->type), @@ -108,11 +116,14 @@ void compare_http_result(int const base_cl, json_data_t const &prev_http_result, } else { if (record_type != iter->type) { record_type = iter->type; - for (auto current_range = eq_range.first; - current_range != eq_range.second; ++current_range) { - spdlog::info("[NEW][{}][{}] `{}`", iter->domain_name, - dns_record_type_to_str(current_range->type), - current_range->rdata); + bool const any_filter = flag_first_seen || flag_last_seen; + if (!any_filter || flag_first_seen) { + for (auto current_range = eq_range.first; + current_range != eq_range.second; ++current_range) { + spdlog::info("[NEW][{}][{}] `{}`", iter->domain_name, + dns_record_type_to_str(current_range->type), + current_range->rdata); + } } } } @@ -127,9 +138,12 @@ void compare_http_result(int const base_cl, json_data_t const &prev_http_result, case_insensitive_compare(a.rdata, b.rdata); }); if (!found) { - spdlog::info("[NEW][{}][{}] `{}`", iter->domain_name, - dns_record_type_to_str(current_elem.type), - current_elem.rdata); + bool const any_filter = flag_first_seen || flag_last_seen; + if (!any_filter || flag_first_seen) { + spdlog::info("[NEW][{}][{}] `{}`", iter->domain_name, + dns_record_type_to_str(current_elem.type), + current_elem.rdata); + } } } } @@ -288,6 +302,26 @@ bool read_input_file(cli_args_t const &cli_args, runtime_args_t &rt_args) { return false; } +void merge_tracking_fields(map_container_t &result_map, + std::vector const &previous_data) { + for (auto &[domain, response] : result_map.result()) { + for (auto &record : response.dns_result_list_) { + for (auto const &prev : previous_data) { + if (prev.domain_name == domain && prev.type == record.type && + case_insensitive_compare(prev.rdata, record.rdata)) { + if (prev.first_seen != 0) { + record.first_seen = prev.first_seen; + } + if (prev.seen_count != 0) { + record.seen_count += prev.seen_count; + } + break; + } + } + } + } +} + void start_name_checking(runtime_args_t &&rt_args) { std::size_t const user_specified_thread = (rt_args.thread_count > 0) ? (std::size_t)rt_args.thread_count @@ -350,6 +384,9 @@ void start_name_checking(runtime_args_t &&rt_args) { } thread_pool->join(); } + if (rt_args.previous_data) { + merge_tracking_fields(result_map, *rt_args.previous_data); + } if (!silent) { spdlog::info("Writing JSON output"); } diff --git a/dooked/source/main.cpp b/dooked/source/main.cpp index cf29460..dec8fd3 100644 --- a/dooked/source/main.cpp +++ b/dooked/source/main.cpp @@ -9,6 +9,8 @@ bool no_bytes_count = false; bool silent = false; bool compare_cl = false; +bool flag_first_seen = false; +bool flag_last_seen = false; int main(int argc, char **argv) { CLI::App app{"dooked -- a CLI tool to enumerate DNS info"}; @@ -47,6 +49,10 @@ int main(int argc, char **argv) { "program returns 0 as the content-length as opposed the total" "bytes returned from the call to I/O socket read"); app.add_flag("--silent", silent, "do not show any DNS or HTTP info"); + app.add_flag("--fs", flag_first_seen, + "show only first-seen (new) DNS records; suppresses last-seen"); + app.add_flag("--ls", flag_last_seen, + "show only last-seen (missing/removed) DNS records; suppresses first-seen"); CLI11_PARSE(app, argc, argv); diff --git a/dooked/source/utils/io_utils.cpp b/dooked/source/utils/io_utils.cpp index a1bd5d3..91ca09d 100644 --- a/dooked/source/utils/io_utils.cpp +++ b/dooked/source/utils/io_utils.cpp @@ -6,6 +6,22 @@ void to_json(json &j, probe_result_t const &record) { j = json{{"ttl", record.ttl}, {"type", dns_record_type_to_str(record.type)}, {"info", record.rdata}}; + if constexpr (has_tracking_fields_v) { + j["first-seen"] = record.first_seen; + j["last-seen"] = record.last_seen; + j["seen"] = record.seen_count; + } +} + +void from_json(json const &j, probe_result_t &record) { + record.ttl = j.value("ttl", std::uint32_t{0}); + record.type = dns_str_to_record_type(j.value("type", std::string{})); + record.rdata = j.value("info", std::string{}); + if constexpr (has_tracking_fields_v) { + record.first_seen = j.value("first-seen", std::uint64_t{0}); + record.last_seen = j.value("last-seen", std::uint64_t{0}); + record.seen_count = j.value("seen", std::uint32_t{0}); + } } bool is_text_file(std::string const &file_extension) {