strip Content-Length header to prevent duplicate

Add Max-Age to checkpoint-token cookie, so that the cookie persists
after the end of the current session.

README.md

@@ -15,6 +15,7 @@ AI scrapers are everywhere. This will stop them. `robots.txt` won't.
 - Support for IP-Range based rules (both ipv4 and ipv6)
 - Support for async (multithreaded) request handling
 - Minimal. The waiting page is tiny and light on network usage.
+- Support for verbose traffic logging for later inspection or statistics
 
 ### Planned features
 - Dynamic challenge amount (aka difficulty) based on traffic

example/config.jsonc

@@ -69,5 +69,24 @@
     "async_proxy": true,
 
     // If enabled, specific requests that look like git HTTP(s) clones will be let through.
-    "git_host": false
+    "git_host": false,
+
+    // If enabled, fingerprinting is ignored and any IP will be able to use
+    // the cached token as long as it is still valid.
+    "ignore_fingerprinting": false,
+
+    // Traffic logging to a .csv file
+    "logging": {
+        "log_traffic": false,
+
+        // This is a sample schema with all supported fields
+        // Please keep in mind your local legal regulations, as IPs under GDPR are considered personal data.
+        "traffic_log_schema": "epoch,ip,domain,resource,useragent,action",
+
+        // Where to save the logfile. Each run will continue appending to this file. It may grow HUGE! No automatic pruning / compression is done.
+        "traffic_log_file": "./traffic.csv"
+    },
+
+    // how long the token (solved challenge) should be valid for before showing a new challenge, in seconds
+    "token_valid_for": 3600 // 1 hour
 }

html/index.html

@@ -212,7 +212,7 @@
 
         <noscript>
             <p class="text-description">
-                This website uses <a class="link" href="https://github.com/vaxerski/checkpoint">checkpoint</a> and
+                This website uses&nbsp;<a class="link" href="https://github.com/vaxerski/checkpoint">checkpoint</a>&nbsp;and
                 requires
                 javascript to
                 automatically verify you are not a bot.<br /><br />
@@ -227,7 +227,7 @@
 
             <form action="/checkpoint/challengeNoJs" method="get">
                 <div>
-                    <label class="form-label" for="name">Enter the output: </label>
+                    <label class="form-label" for="name">Enter the output </label>
                     <input class="form-input" type="text" name="solution" id="solution" required />
                 </div>
                 <input class="input-hidden" type="text" name="fingerprint" id="fingerprint"
@@ -249,21 +249,47 @@
 
             <style>
                 .form-label {
-                    color: #fff;
+                    color: #d9d9d9;
                     display: block;
                     text-align: center;
-                }
+                    margin-bottom: 0.5rem;
-
+                    font-size: 1rem;
-                .form-button {
-                    margin-left: 9.5rem;
-                    width: calc(100%-19rem);
-                    display: block;
-                    position: relative;
                 }
 
                 .form-input {
-                    margin-left: 9.5rem;
+                    display: block;
-                    width: calc(100%-19rem);
+                    margin: 0 auto;
+                    width: 15rem;
+                    background-color: #161616;
+                    color: #d9d9d9;
+                    border: 1px solid #280d0e;
+                    border-radius: 0.3rem;
+                    padding: 0.5rem;
+                    font-size: 1rem;
+                    margin-bottom: 0.5rem;
+                    outline: none;
+                    transition: border-color 0.2s ease-in-out;
+                }
+
+                .form-input:focus {
+                    border-color: #692225;
+                }
+
+                .form-button {
+                    display: block;
+                    margin: 0 auto;
+                    background-color: #280d0e;
+                    color: #d9d9d9;
+                    border: none;
+                    border-radius: 0.3rem;
+                    padding: 0.5rem 1.5rem;
+                    font-size: 1rem;
+                    cursor: pointer;
+                    transition: background-color 0.2s ease-in-out;
+                }
+
+                .form-button:hover {
+                    background-color: #692225;
                 }
 
                 .code-block {

src/config/Config.cpp

@@ -38,7 +38,7 @@ CConfig::CConfig() {
     for (const auto& ic : m_config.rules) {
         CConfigRule rule;
         rule.action = strToAction(ic.action);
-        
+
         if (ic.difficulty != -1)
             rule.difficulty = ic.difficulty;
 

src/config/Config.hpp

@@ -35,8 +35,16 @@ class CConfig {
         bool                     trace_logging                = false;
         std::vector<SConfigRule> rules                        = {};
         int                      default_challenge_difficulty = 4;
+        int                      token_valid_for              = 3600;
+        bool                     ignore_fingerprinting        = false;
         bool                     async_proxy                  = true;
         std::vector<SProxyRule>  proxy_rules;
+
+        struct {
+            bool        log_traffic = false;
+            std::string traffic_log_schema;
+            std::string traffic_log_file;
+        } logging;
     } m_config;
 
     struct {

src/core/Challenge.cpp

@@ -49,17 +49,12 @@ CChallenge::CChallenge(const std::string& jsonResponse) {
 CChallenge::CChallenge(const Pistache::Http::Request& reqResponse) {
     auto& q = reqResponse.query();
 
-    if (!q.has("solution")
+    if (!q.has("solution") || !q.has("fingerprint") || !q.has("challenge") || !q.has("timestamp") || !q.has("sig") || !q.has("difficulty"))
-        || !q.has("fingerprint")
-        || !q.has("challenge")
-        || !q.has("timestamp")
-        || !q.has("sig")
-        || !q.has("difficulty"))
         return;
 
-    m_challenge = q.get("challenge").value();
+    m_challenge   = q.get("challenge").value();
     m_fingerprint = q.get("fingerprint").value();
-    m_sig = q.get("sig").value();
+    m_sig         = q.get("sig").value();
 
     try {
         m_issued = std::chrono::system_clock::time_point(std::chrono::seconds(std::stoull(q.get("timestamp").value())));

src/core/Handler.cpp

@@ -14,6 +14,8 @@
 #include "../GlobalState.hpp"
 #include "../config/Config.hpp"
 #include "../helpers/FsUtils.hpp"
+#include "../helpers/RequestUtils.hpp"
+#include "../logging/TrafficLogger.hpp"
 
 #include <filesystem>
 #include <random>
@@ -25,8 +27,7 @@
 #include <openssl/evp.h>
 #include <magic.h>
 
-constexpr const uint64_t TOKEN_MAX_AGE_MS  = 1000 * 60 * 60; // 1hr
+constexpr const char* TOKEN_COOKIE_NAME = "checkpoint-token";
-constexpr const char*    TOKEN_COOKIE_NAME = "checkpoint-token";
 
 //
 
@@ -56,73 +57,10 @@ static std::string generateToken() {
     return ss.str();
 }
 
-std::string CServerHandler::fingerprintForRequest(const Pistache::Http::Request& req) {
-    const auto                                                    HEADERS = req.headers();
-    std::shared_ptr<const Pistache::Http::Header::AcceptEncoding> acceptEncodingHeader;
-    std::shared_ptr<const Pistache::Http::Header::UserAgent>      userAgentHeader;
-    std::shared_ptr<const AcceptLanguageHeader>                   languageHeader;
-
-    std::string                                                   input = "checkpoint-";
-
-    try {
-        acceptEncodingHeader = Pistache::Http::Header::header_cast<Pistache::Http::Header::AcceptEncoding>(HEADERS.get("Accept-Encoding"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    try {
-        languageHeader = Pistache::Http::Header::header_cast<AcceptLanguageHeader>(HEADERS.get("Accept-Language"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    try {
-        userAgentHeader = Pistache::Http::Header::header_cast<Pistache::Http::Header::UserAgent>(HEADERS.get("User-Agent"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    input += ipForRequest(req);
-    // TODO: those seem to change. Find better things to hash.
-    // if (acceptEncodingHeader)
-    //     input += HEADERS.getRaw("Accept-Encoding").value();
-    // if (languageHeader)
-    //     input += languageHeader->language();
-    if (userAgentHeader)
-        input += userAgentHeader->agent();
-
-    return g_pCrypto->sha256(input);
-}
-
 bool CServerHandler::isResourceCheckpoint(const std::string_view& res) {
     return res.starts_with("/checkpoint/");
 }
 
-std::string CServerHandler::ipForRequest(const Pistache::Http::Request& req) {
-    std::shared_ptr<const CFConnectingIPHeader> cfHeader;
-    std::shared_ptr<const XRealIPHeader>        xRealIPHeader;
-
-    try {
-        cfHeader = Pistache::Http::Header::header_cast<CFConnectingIPHeader>(req.headers().get("cf-connecting-ip"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    try {
-        xRealIPHeader = Pistache::Http::Header::header_cast<XRealIPHeader>(req.headers().get("X-Real-IP"));
-    } catch (std::exception& e) {
-        ; // silent ignore
-    }
-
-    if (cfHeader)
-        return cfHeader->ip();
-
-    if (xRealIPHeader)
-        return xRealIPHeader->ip();
-
-    return req.address().host();
-}
-
 void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter response) {
     const auto                                                 HEADERS = req.headers();
     std::shared_ptr<const Pistache::Http::Header::Host>        hostHeader;
@@ -186,7 +124,7 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
 
     Debug::log(LOG, "New request: {}:{}{}", hostHeader->host(), hostHeader->port().toString(), req.resource());
 
-    const auto REQUEST_IP = ipForRequest(req);
+    const auto REQUEST_IP = NRequestUtils::ipForRequest(req);
 
     Debug::log(LOG, " | Request author: IP {}, direct: {}", REQUEST_IP, req.address().host());
 
@@ -228,12 +166,14 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
                 Debug::log(TRACE, "Request looks like it is coming from git (UA + GP). Accepting.");
 
                 proxyPass(req, response);
+                g_pTrafficLogger->logTraffic(req, "PASS (git)");
                 return;
             } else if (userAgentHeader->agent().starts_with("git/")) {
                 Debug::log(LOG, " | Action: PASS (git)");
                 Debug::log(TRACE, "Request looks like it is coming from git (UA git). Accepting.");
 
                 proxyPass(req, response);
+                g_pTrafficLogger->logTraffic(req, "PASS (git)");
                 return;
             }
         }
@@ -249,10 +189,12 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
                     case IP_ACTION_DENY:
                         Debug::log(LOG, " | Action: DENY (rule)");
                         response.send(Pistache::Http::Code::Forbidden, "Blocked by checkpoint");
+                        g_pTrafficLogger->logTraffic(req, "DENY (rule)");
                         return;
                     case IP_ACTION_ALLOW:
                         Debug::log(LOG, " | Action: PASS (rule)");
                         proxyPass(req, response);
+                        g_pTrafficLogger->logTraffic(req, "PASS (rule)");
                         return;
                     case IP_ACTION_CHALLENGE:
                         Debug::log(LOG, " | Action: CHALLENGE (rule)");
@@ -273,12 +215,13 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
         if (TOKEN.valid()) {
             const auto AGE = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count() -
                 std::chrono::duration_cast<std::chrono::milliseconds>(TOKEN.issued().time_since_epoch()).count();
-            if (AGE <= TOKEN_MAX_AGE_MS && TOKEN.fingerprint() == fingerprintForRequest(req)) {
+            if (AGE <= g_pConfig->m_config.token_valid_for * 1000 && (TOKEN.fingerprint() == NRequestUtils::fingerprintForRequest(req) || g_pConfig->m_config.ignore_fingerprinting)) {
                 Debug::log(LOG, " | Action: PASS (token)");
+                g_pTrafficLogger->logTraffic(req, "PASS (token)");
                 proxyPass(req, response);
                 return;
             } else { // token has been used from a different IP or is expired. Nuke it.
-                if (AGE > TOKEN_MAX_AGE_MS)
+                if (AGE > g_pConfig->m_config.token_valid_for * 1000)
                     Debug::log(LOG, " | Action: CHALLENGE (token expired)");
                 else
                     Debug::log(LOG, " | Action: CHALLENGE (token fingerprint mismatch)");
@@ -295,7 +238,7 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
         const auto        PATH_RAW      = NFsUtils::htmlPath(RESOURCE_PATH);
 
         std::error_code   ec;
-        auto        PATH_ABSOLUTE = std::filesystem::canonical(PATH_RAW, ec);
+        auto              PATH_ABSOLUTE = std::filesystem::canonical(PATH_RAW, ec);
 
         if (ec) {
             // bad resource, try .html
@@ -305,12 +248,14 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
         if (ec) {
             // bad resource
             response.send(Pistache::Http::Code::Bad_Request, "Bad Request");
+            g_pTrafficLogger->logTraffic(req, "BAD_CHECKPOINT_RESOURCE");
             return;
         }
 
         if (!PATH_ABSOLUTE.string().starts_with(HTML_ROOT)) {
             // directory traversal
             response.send(Pistache::Http::Code::Bad_Request, "Bad Request");
+            g_pTrafficLogger->logTraffic(req, "BAD_CHECKPOINT_RESOURCE");
             return;
         }
 
@@ -326,9 +271,12 @@ void CServerHandler::onRequest(const Pistache::Http::Request& req, Pistache::Htt
 
         auto body = NFsUtils::readFileAsString(PATH_ABSOLUTE).value_or("");
         response.send(body.empty() ? Pistache::Http::Code::Internal_Server_Error : Pistache::Http::Code::Ok, body);
+        g_pTrafficLogger->logTraffic(req, "PASS (Checkpoint resource)");
         return;
     }
 
+    g_pTrafficLogger->logTraffic(req, "CHALLENGE");
+
     serveStop(req, response, challengeDifficulty);
 }
 
@@ -338,7 +286,7 @@ void CServerHandler::onTimeout(const Pistache::Http::Request& request, Pistache:
 
 void CServerHandler::challengeSubmitted(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, bool js) {
     const auto JSON        = req.body();
-    const auto FINGERPRINT = fingerprintForRequest(req);
+    const auto FINGERPRINT = NRequestUtils::fingerprintForRequest(req);
 
     CChallenge CHALLENGE;
     if (!js)
@@ -348,6 +296,7 @@ void CServerHandler::challengeSubmitted(const Pistache::Http::Request& req, Pist
 
     if (!CHALLENGE.valid()) {
         response.send(Pistache::Http::Code::Bad_Request, "Bad request");
+        g_pTrafficLogger->logTraffic(req, "CHALLENGE_FAIL");
         return;
     }
 
@@ -368,7 +317,7 @@ void CServerHandler::challengeSubmitted(const Pistache::Http::Request& req, Pist
     }
 
     response.headers().add(
-        std::make_shared<SetCookieHeader>(std::string{TOKEN_COOKIE_NAME} + "=" + TOKEN.tokenCookie() + "; Domain=" + hostDomain + "; HttpOnly; Path=/; Secure; SameSite=Lax"));
+        std::make_shared<SetCookieHeader>(std::string{TOKEN_COOKIE_NAME} + "=" + TOKEN.tokenCookie() + "; Domain=" + hostDomain + "; Max-Age=" + std::to_string(g_pConfig->m_config.token_valid_for) + "; HttpOnly; Path=/; Secure; SameSite=Lax"));
 
     if (js)
         response.send(Pistache::Http::Code::Ok, "Ok");
@@ -376,6 +325,8 @@ void CServerHandler::challengeSubmitted(const Pistache::Http::Request& req, Pist
         response.headers().add<Pistache::Http::Header::Location>("/");
         response.send(Pistache::Http::Code::Moved_Permanently, "");
     }
+
+    g_pTrafficLogger->logTraffic(req, "CHALLENGE_PASS");
 }
 
 void CServerHandler::serveStop(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, int difficulty) {
@@ -385,7 +336,7 @@ void CServerHandler::serveStop(const Pistache::Http::Request& req, Pistache::Htt
     page.setTemplateRoot(PAGE_ROOT);
 
     const auto NONCE     = generateNonce();
-    const auto CHALLENGE = CChallenge(fingerprintForRequest(req), NONCE, difficulty);
+    const auto CHALLENGE = CChallenge(NRequestUtils::fingerprintForRequest(req), NONCE, difficulty);
 
     auto       hostDomain = req.headers().getRaw("Host").value();
     if (hostDomain.contains(":"))
@@ -496,7 +447,7 @@ void CServerHandler::proxyPassInternal(const Pistache::Http::Request& req, Pista
             const auto HEADERSRESP = resp.headers().list();
 
             for (auto& h : HEADERSRESP) {
-                if (std::string_view{h->name()} == "Transfer-Encoding") {
+                if (std::string_view{h->name()} == "Transfer-Encoding" || std::string_view{h->name()} == "Content-Length") {
                     Debug::log(TRACE, "Header out: {}: {} (DROPPED)", h->name(), resp.headers().getRaw(h->name()).value());
                     continue;
                 }
@@ -532,4 +483,4 @@ void CServerHandler::proxyPassInternal(const Pistache::Http::Request& req, Pista
     b.wait_for(std::chrono::seconds(g_pConfig->m_config.proxy_timeout_sec));
 
     client.shutdown();
-}
+}

src/core/Handler.hpp

@@ -17,15 +17,13 @@ class CServerHandler : public Pistache::Http::Handler {
     void onTimeout(const Pistache::Http::Request& request, Pistache::Http::ResponseWriter response);
 
   private:
-    void        serveStop(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, int difficulty);
+    void serveStop(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, int difficulty);
-    void        proxyPass(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response);
+    void proxyPass(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response);
-    void        proxyPassInternal(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, bool async = false);
+    void proxyPassInternal(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, bool async = false);
-    void        proxyPassAsync(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response);
+    void proxyPassAsync(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response);
-    void        challengeSubmitted(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, bool js);
+    void challengeSubmitted(const Pistache::Http::Request& req, Pistache::Http::ResponseWriter& response, bool js);
-    std::string fingerprintForRequest(const Pistache::Http::Request& req);
-    std::string ipForRequest(const Pistache::Http::Request& req);
 
-    bool        isResourceCheckpoint(const std::string_view& res);
+    bool isResourceCheckpoint(const std::string_view& res);
 
     struct SChallengeResponse {
         std::string       challenge;

src/headers/csrfHeader.hpp

@@ -0,0 +1,24 @@
+#include <pistache/http_headers.h>
+#include <pistache/net.h>
+
+class XCSRFTokenHeader : public Pistache::Http::Header::Header {
+  public:
+    NAME("X-Csrf-Token");
+
+    XCSRFTokenHeader() = default;
+
+    void parse(const std::string& str) override {
+        m_data = str;
+    }
+
+    void write(std::ostream& os) const override {
+        os << m_data;
+    }
+
+    std::string ip() const {
+        return m_data;
+    }
+
+  private:
+    std::string m_data = "";
+};

src/helpers/RequestUtils.cpp

@@ -0,0 +1,75 @@
+#include "RequestUtils.hpp"
+
+#include "../core/Crypto.hpp"
+
+#include "../headers/authorization.hpp"
+#include "../headers/cfHeader.hpp"
+#include "../headers/xforwardfor.hpp"
+#include "../headers/gitProtocolHeader.hpp"
+#include "../headers/wwwAuthenticateHeader.hpp"
+#include "../headers/acceptLanguageHeader.hpp"
+#include "../headers/setCookieHeader.hpp"
+#include "../headers/xrealip.hpp"
+
+std::string NRequestUtils::fingerprintForRequest(const Pistache::Http::Request& req) {
+    const auto                                                    HEADERS = req.headers();
+    std::shared_ptr<const Pistache::Http::Header::AcceptEncoding> acceptEncodingHeader;
+    std::shared_ptr<const Pistache::Http::Header::UserAgent>      userAgentHeader;
+    std::shared_ptr<const AcceptLanguageHeader>                   languageHeader;
+
+    std::string                                                   input = "checkpoint-";
+
+    try {
+        acceptEncodingHeader = Pistache::Http::Header::header_cast<Pistache::Http::Header::AcceptEncoding>(HEADERS.get("Accept-Encoding"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    try {
+        languageHeader = Pistache::Http::Header::header_cast<AcceptLanguageHeader>(HEADERS.get("Accept-Language"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    try {
+        userAgentHeader = Pistache::Http::Header::header_cast<Pistache::Http::Header::UserAgent>(HEADERS.get("User-Agent"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    input += ipForRequest(req);
+    // TODO: those seem to change. Find better things to hash.
+    // if (acceptEncodingHeader)
+    //     input += HEADERS.getRaw("Accept-Encoding").value();
+    // if (languageHeader)
+    //     input += languageHeader->language();
+    if (userAgentHeader)
+        input += userAgentHeader->agent();
+
+    return g_pCrypto->sha256(input);
+}
+
+std::string NRequestUtils::ipForRequest(const Pistache::Http::Request& req) {
+    std::shared_ptr<const CFConnectingIPHeader> cfHeader;
+    std::shared_ptr<const XRealIPHeader>        xRealIPHeader;
+
+    try {
+        cfHeader = Pistache::Http::Header::header_cast<CFConnectingIPHeader>(req.headers().get("cf-connecting-ip"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    try {
+        xRealIPHeader = Pistache::Http::Header::header_cast<XRealIPHeader>(req.headers().get("X-Real-IP"));
+    } catch (std::exception& e) {
+        ; // silent ignore
+    }
+
+    if (cfHeader)
+        return cfHeader->ip();
+
+    if (xRealIPHeader)
+        return xRealIPHeader->ip();
+
+    return req.address().host();
+}

src/helpers/RequestUtils.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <string>
+
+#include <pistache/http.h>
+
+namespace NRequestUtils {
+    std::string fingerprintForRequest(const Pistache::Http::Request& req);
+    std::string ipForRequest(const Pistache::Http::Request& req);
+};

src/logging/TrafficLogger.cpp

@@ -0,0 +1,132 @@
+#include "TrafficLogger.hpp"
+
+#include <sstream>
+#include <fmt/format.h>
+
+#include "../config/Config.hpp"
+#include "../debug/log.hpp"
+#include "../helpers/RequestUtils.hpp"
+
+CTrafficLogger::CTrafficLogger() {
+    if (!g_pConfig->m_config.logging.log_traffic)
+        return;
+
+    const auto COMMAS = std::count(g_pConfig->m_config.logging.traffic_log_schema.begin(), g_pConfig->m_config.logging.traffic_log_schema.end(), ',');
+
+    // parse the schema
+    std::string_view curr;
+    size_t           lastPos = 0;
+    bool             first   = true;
+    auto             advance = [&]() {
+        size_t prev = !first ? lastPos + 1 : lastPos;
+        lastPos     = g_pConfig->m_config.logging.traffic_log_schema.find(',', prev);
+
+        if (lastPos == std::string::npos)
+            curr = std::string_view{g_pConfig->m_config.logging.traffic_log_schema}.substr(prev);
+        else
+            curr = std::string_view{g_pConfig->m_config.logging.traffic_log_schema}.substr(prev, lastPos - prev);
+
+        first = false;
+    };
+
+    for (size_t i = 0; i < COMMAS + 1; ++i) {
+        advance();
+
+        if (curr == "ip")
+            m_logSchema.emplace_back(TRAFFIC_IP);
+        else if (curr == "epoch")
+            m_logSchema.emplace_back(TRAFFIC_EPOCH);
+        else if (curr == "domain")
+            m_logSchema.emplace_back(TRAFFIC_DOMAIN);
+        else if (curr == "resource")
+            m_logSchema.emplace_back(TRAFFIC_RESOURCE);
+        else if (curr == "useragent")
+            m_logSchema.emplace_back(TRAFFIC_USERAGENT);
+        else if (curr == "action")
+            m_logSchema.emplace_back(TRAFFIC_ACTION);
+
+        if (curr == "")
+            break;
+    }
+
+    m_file.open(g_pConfig->m_config.logging.traffic_log_file, std::ios::app);
+
+    if (!m_file.good())
+        Debug::die("TrafficLogger: bad file {}", g_pConfig->m_config.logging.traffic_log_file);
+}
+
+CTrafficLogger::~CTrafficLogger() {
+    if (m_file.is_open())
+        m_file.close();
+}
+
+static std::string sanitize(const std::string& s) {
+    if (s.empty())
+        return s;
+
+    std::string cpy = s;
+    size_t      pos = 0;
+    while ((pos = cpy.find('"', pos)) != std::string::npos) {
+        cpy.replace(pos, 1, "\\\"");
+        pos += 2;
+    }
+
+    return cpy;
+}
+
+void CTrafficLogger::logTraffic(const Pistache::Http::Request& req, const char* actionTaken) {
+    if (!g_pConfig->m_config.logging.log_traffic)
+        return;
+
+    std::stringstream ss;
+
+    for (const auto& t : m_logSchema) {
+        switch (t) {
+            case TRAFFIC_EPOCH: {
+                ss << fmt::format("{},", std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count());
+                break;
+            }
+
+            case TRAFFIC_DOMAIN: {
+                const auto HOST = Pistache::Http::Header::header_cast<Pistache::Http::Header::Host>(req.headers().get("Host"));
+                ss << fmt::format("\"{}\",", sanitize(HOST->host()));
+                break;
+            }
+
+            case TRAFFIC_IP: {
+                ss << fmt::format("{},", NRequestUtils::ipForRequest(req));
+                break;
+            }
+
+            case TRAFFIC_RESOURCE: {
+                ss << fmt::format("\"{}\",", sanitize(req.resource()));
+                break;
+            }
+
+            case TRAFFIC_USERAGENT: {
+                if (!req.headers().has("User-Agent")) {
+                    ss << "\"<no data>\",";
+                    break;
+                }
+                const auto UA = Pistache::Http::Header::header_cast<Pistache::Http::Header::UserAgent>(req.headers().get("User-Agent"));
+                ss << fmt::format("\"{}\",", sanitize(UA->agent()));
+                break;
+            }
+
+            case TRAFFIC_ACTION: {
+                ss << fmt::format("{},", actionTaken);
+                break;
+            }
+        }
+    }
+
+    std::string trafficLine = ss.str();
+    if (trafficLine.empty())
+        return;
+
+    // replace , with \n
+    trafficLine.back() = '\n';
+
+    m_file << trafficLine;
+    m_file.flush();
+}

src/logging/TrafficLogger.hpp

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <string>
+#include <cstdint>
+#include <memory>
+#include <fstream>
+
+#include "../config/ConfigTypes.hpp"
+
+#include <pistache/http.h>
+
+class CTrafficLogger {
+  public:
+    CTrafficLogger();
+    ~CTrafficLogger();
+
+    void logTraffic(const Pistache::Http::Request& req, const char* actionTaken);
+
+  private:
+    enum eTrafficLoggerProps : uint8_t {
+        TRAFFIC_EPOCH = 0,
+        TRAFFIC_IP,
+        TRAFFIC_DOMAIN,
+        TRAFFIC_RESOURCE,
+        TRAFFIC_USERAGENT,
+        TRAFFIC_ACTION,
+    };
+
+    std::vector<eTrafficLoggerProps> m_logSchema;
+    std::ofstream                    m_file;
+};
+
+inline std::unique_ptr<CTrafficLogger> g_pTrafficLogger;

src/main.cpp

@@ -16,6 +16,7 @@
 #include "headers/acceptLanguageHeader.hpp"
 #include "headers/setCookieHeader.hpp"
 #include "headers/xrealip.hpp"
+#include "headers/csrfHeader.hpp"
 
 #include "debug/log.hpp"
 #include "helpers/FsUtils.hpp"
@@ -25,6 +26,8 @@
 
 #include "config/Config.hpp"
 
+#include "logging/TrafficLogger.hpp"
+
 #include "GlobalState.hpp"
 
 #include <signal.h>
@@ -86,8 +89,10 @@ int main(int argc, char** argv, char** envp) {
     Pistache::Http::Header::Registry::instance().registerHeader<AcceptLanguageHeader>();
     Pistache::Http::Header::Registry::instance().registerHeader<SetCookieHeader>();
     Pistache::Http::Header::Registry::instance().registerHeader<XRealIPHeader>();
+    Pistache::Http::Header::Registry::instance().registerHeader<XCSRFTokenHeader>();
 
-    g_pCrypto = std::make_unique<CCrypto>();
+    g_pCrypto        = std::make_unique<CCrypto>();
+    g_pTrafficLogger = std::make_unique<CTrafficLogger>();
 
     auto endpoint = std::make_unique<Pistache::Http::Endpoint>(address);
     auto opts     = Pistache::Http::Endpoint::options().threads(threads).flags(Pistache::Tcp::Options::ReuseAddr | Pistache::Tcp::Options::ReusePort);