feat: add userspace network stack

.github/workflows/check.yaml

@@ -7,14 +7,20 @@ on:
 jobs:
   format:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        path:
+          - 'candy'
+          - 'candy-cli'
+          - 'candy-service'
     steps:
       - name: checkout
         uses: actions/checkout@v4
       - name: check format
         uses: jidicula/clang-format-action@v4.11.0
         with:
-          check-path: 'src'
-          exclude-regex: 'argparse.h'
+          check-path: ${{ matrix.path }}
+          exclude-regex: '(argparse.h|lwip)'
 
   linux:
     runs-on: ubuntu-latest

CMakeLists.txt

@@ -91,6 +91,11 @@ else()
     find_package(Poco REQUIRED COMPONENTS Foundation XML JSON Net NetSSL Util)
 endif()
 
+set(LWIP_DIR ${CMAKE_CURRENT_SOURCE_DIR}/candy/src/netstack/lwip)
+set(LWIP_INCLUDE_DIRS ${LWIP_DIR}/src/include)
+include(${LWIP_DIR}/src/Filelists.cmake)
+target_include_directories(lwipcore PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/candy/src/netstack)
+
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 

candy-cli/src/config.cc

@@ -53,8 +53,7 @@ int arguments::parse(int argc, char *argv[]) {
     program.add_argument("-w", "--websocket")
         .help("WebSocket signaling address (e.g. \"ws://host:port/ws\").\n"
               "Client supports ws:// and wss://; server supports ws:// only.")
-        .metavar("<url>")
-        .required();
+        .metavar("<url>");
 
     program.add_argument("-p", "--password")
         .help("pre-shared key for authentication and P2P encryption.\n"

candy/CMakeLists.txt

@@ -50,6 +50,11 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
     include_directories(${CMAKE_CURRENT_BINARY_DIR}/wintun/include)
 endif()
 
+set(LWIP_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/netstack/lwip)
+target_link_libraries(candy-library PRIVATE lwipcore)
+target_include_directories(candy-library PRIVATE ${LWIP_DIR}/src/include)
+target_include_directories(candy-library PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/netstack)
+
 set_target_properties(candy-library PROPERTIES OUTPUT_NAME "candy")
 
 add_library(Candy::Library ALIAS candy-library)

candy/src/core/client.cc

@@ -62,6 +62,10 @@ MsgQueue &Client::getWsMsgQueue() {
     return this->wsMsgQueue;
 }
 
+MsgQueue &Client::getNetstackMsgQueue() {
+    return this->netstackMsgQueue;
+}
+
 void Client::setPassword(const std::string &password) {
     ws.setPassword(password);
     peerManager.setPassword(password);
@@ -107,6 +111,10 @@ void Client::setMtu(int mtu) {
     tun.setMTU(mtu);
 }
 
+int Client::getMtu() {
+    return tun.getMTU();
+}
+
 void Client::run() {
     this->running.store(true);
 
@@ -119,14 +127,19 @@ void Client::run() {
     if (peerManager.run(this)) {
         return;
     }
+    if (netstack.run(this)) {
+        return;
+    }
 
     ws.wait();
     tun.wait();
     peerManager.wait();
+    netstack.wait();
 
     wsMsgQueue.clear();
     tunMsgQueue.clear();
     peerMsgQueue.clear();
+    netstackMsgQueue.clear();
 }
 
 bool Client::isRunning() {

candy/src/core/client.h

@@ -3,6 +3,7 @@
 #define CANDY_CORE_CLIENT_H
 
 #include "core/message.h"
+#include "netstack/netstack.h"
 #include "peer/manager.h"
 #include "tun/tun.h"
 #include "utils/atomic.h"
@@ -38,6 +39,7 @@ class Client {
     void setPort(int port);
     void setLocalhost(std::string ip);
     void setMtu(int mtu);
+    int getMtu();
 
     void setExptTunAddress(const std::string &cidr);
     void setVirtualMac(const std::string &vmac);
@@ -57,13 +59,15 @@ class Client {
     MsgQueue &getTunMsgQueue();
     MsgQueue &getPeerMsgQueue();
     MsgQueue &getWsMsgQueue();
+    MsgQueue &getNetstackMsgQueue();
 
 private:
-    MsgQueue tunMsgQueue, peerMsgQueue, wsMsgQueue;
+    MsgQueue tunMsgQueue, peerMsgQueue, wsMsgQueue, netstackMsgQueue;
 
     Tun tun;
     PeerManager peerManager;
     WebSocketClient ws;
+    Netstack netstack;
 
 private:
     std::string tunName;

candy/src/core/common.cc

@@ -13,4 +13,4 @@ std::string create_vmac() {
     return randomHexString(VMAC_SIZE);
 }
 
-} // namespace candy
+} // namespace candy

candy/src/core/net.cc

@@ -10,6 +10,13 @@ IP4::IP4(const std::string &ip) {
     fromString(ip);
 }
 
+IP4::IP4(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3) {
+    raw[0] = b0;
+    raw[1] = b1;
+    raw[2] = b2;
+    raw[3] = b3;
+}
+
 IP4 IP4::operator=(const std::string &ip) {
     fromString(ip);
     return *this;

candy/src/core/net.h

@@ -48,6 +48,7 @@ template <typename T> T hton(T v) {
 class __attribute__((packed)) IP4 {
 public:
     IP4(const std::string &ip = "0.0.0.0");
+    IP4(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3);
     IP4 operator=(const std::string &ip);
     IP4 operator&(IP4 another) const;
     IP4 operator|(IP4 another) const;
@@ -120,6 +121,19 @@ class Address {
 
 } // namespace candy
 
+template <> struct fmt::formatter<candy::IP4> : fmt::formatter<std::string> {
+    template <typename FormatContext> auto format(const candy::IP4 &ip, FormatContext &ctx) const {
+        return fmt::formatter<std::string>::format(ip.toString(), ctx);
+    }
+};
+
+namespace candy::netstack {
+constexpr uint16_t TAG_MASK = 0x0003;
+constexpr uint16_t TAG_UNSURE = 0;
+constexpr uint16_t TAG_REQUEST = 1;
+constexpr uint16_t TAG_RESPONSE = 2;
+} // namespace candy::netstack
+
 namespace std {
 using candy::IP4;
 template <> struct hash<IP4> {

candy/src/netstack/arch/cc.h

@@ -0,0 +1,16 @@
+#ifndef CANDY_ARCH_CC_H
+#define CANDY_ARCH_CC_H
+
+// Include lwipopts.h here so that any change to it triggers
+// recompilation of all lwIP source files via the dependency chain:
+//   lwip/arch.h -> arch/cc.h -> lwipopts.h
+#include "lwipopts.h"
+
+#define LWIP_PROVIDE_ERRNO 1
+
+#define LWIP_NO_STDDEF_H 0
+#define LWIP_NO_STDINT_H 0
+#define LWIP_NO_INTTYPES_H 0
+#define LWIP_NO_LIMITS_H 0
+
+#endif

candy/src/netstack/ip.cc

@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: MIT
+#include "netstack/ip.h"
+
+#include <chrono>
+#include <cstring>
+#include <deque>
+#include <memory>
+#include <spdlog/spdlog.h>
+#include <stdexcept>
+
+extern "C" {
+#include <lwip/init.h>
+#include <lwip/ip4.h>
+#include <lwip/netif.h>
+#include <lwip/pbuf.h>
+#include <lwip/tcp.h>
+#include <lwip/timeouts.h>
+#include <lwip/udp.h>
+
+u32_t sys_now(void) {
+    return (u32_t)std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch())
+        .count();
+}
+}
+
+namespace candy {
+
+struct PbufDeleter {
+    void operator()(struct pbuf *p) const {
+        if (p)
+            pbuf_free(p);
+    }
+};
+using PbufPtr = std::unique_ptr<struct pbuf, PbufDeleter>;
+
+IP4 fromLwipHostOrder(uint32_t hostOrder) {
+    return IP4((hostOrder >> 24) & 0xFF, (hostOrder >> 16) & 0xFF, (hostOrder >> 8) & 0xFF, hostOrder & 0xFF);
+}
+
+IP4 lwipToIP4(const void *addr) {
+    auto *a = (const ip4_addr_t *)addr;
+    return fromLwipHostOrder(lwip_ntohl(ip4_addr_get_u32(a)));
+}
+
+struct NetifDeleter {
+    void operator()(struct netif *nif) const {
+        if (nif) {
+            netif_set_down(nif);
+            netif_remove(nif);
+            free(nif);
+        }
+    }
+};
+
+struct IpStackImpl : public IpStack::Impl {
+    Config cfg;
+    std::unique_ptr<struct netif, NetifDeleter> netif;
+    std::deque<std::string> outQ;
+
+    explicit IpStackImpl(const Config &c);
+    ~IpStackImpl();
+    void send(const std::string &pk);
+    std::optional<std::string> recv();
+    struct netif *getNetif() {
+        return netif.get();
+    }
+    static err_t netifOutput(struct netif *nif, struct pbuf *p, const ip4_addr_t *);
+};
+
+IpStackImpl::IpStackImpl(const Config &c) : cfg(c) {
+    lwip_init();
+
+    ip_addr_t ipaddr, netmask, gw;
+    ip4_addr_set_u32(&ipaddr, (uint32_t)cfg.addr);
+    IP4_ADDR(&netmask, 255, 255, 255, 0);
+    IP4_ADDR(&gw, 198, 18, 0, 1);
+
+    struct netif *rawNetif = (struct netif *)calloc(1, sizeof(struct netif));
+    auto initFn = [](struct netif *nif) -> err_t {
+        auto *self = (IpStackImpl *)nif->state;
+        nif->output = &IpStackImpl::netifOutput;
+        nif->mtu = (u16_t)self->cfg.mtu;
+        nif->flags = NETIF_FLAG_BROADCAST | NETIF_FLAG_ETHARP | NETIF_FLAG_UP | NETIF_FLAG_LINK_UP | NETIF_FLAG_PRETEND;
+        return ERR_OK;
+    };
+    if (!netif_add(rawNetif, &ipaddr, &netmask, &gw, this, initFn, nullptr)) {
+        spdlog::error("netif_add failed");
+        free(rawNetif);
+        throw std::runtime_error("IpStack: netif_add failed");
+    }
+    netif.reset(rawNetif);
+    netif_set_default(netif.get());
+    netif_set_up(netif.get());
+}
+
+IpStackImpl::~IpStackImpl() {}
+
+void IpStackImpl::send(const std::string &pk) {
+    PbufPtr pb(pbuf_alloc(PBUF_RAW, pk.size(), PBUF_POOL));
+    if (!pb) {
+        spdlog::error("pbuf_alloc failed");
+        return;
+    }
+    if (pbuf_take(pb.get(), pk.data(), pk.size()) != ERR_OK) {
+        spdlog::error("pbuf_take failed");
+        return;
+    }
+    err_t ret = ip4_input(pb.get(), netif.get());
+    if (ret != ERR_OK)
+        return;
+    pb.release();
+}
+
+std::optional<std::string> IpStackImpl::recv() {
+    if (outQ.empty())
+        return std::nullopt;
+    auto pkt = std::move(outQ.front());
+    outQ.pop_front();
+    return pkt;
+}
+
+err_t IpStackImpl::netifOutput(struct netif *nif, struct pbuf *p, const ip4_addr_t *) {
+    auto *self = (IpStackImpl *)nif->state;
+    std::string pkt(p->tot_len, 0);
+    pbuf_copy_partial(p, pkt.data(), p->tot_len, 0);
+    self->outQ.push_back(std::move(pkt));
+    return ERR_OK;
+}
+
+IpStack::IpStack() : impl(nullptr) {}
+
+std::unique_ptr<IpStack> IpStack::create(const Config &cfg) {
+    try {
+        auto stack = std::unique_ptr<IpStack>(new IpStack());
+        stack->impl = std::unique_ptr<Impl>(new IpStackImpl(cfg));
+        return stack;
+    } catch (const std::exception &e) {
+        spdlog::error("IpStack::create failed: {}", e.what());
+        return nullptr;
+    }
+}
+
+IpStack::~IpStack() = default;
+
+void IpStack::send(const std::string &p) {
+    static_cast<IpStackImpl *>(impl.get())->send(p);
+}
+
+std::optional<std::string> IpStack::recv() {
+    return static_cast<IpStackImpl *>(impl.get())->recv();
+}
+
+void IpStack::bindTcpPretendListener(struct tcp_pcb *pcb) {
+    auto *self = static_cast<IpStackImpl *>(impl.get());
+    tcp_bind_netif(pcb, self->netif.get());
+    tcp_bind(pcb, NULL, 0);
+}
+
+void IpStack::bindUdpPretendSocket(struct udp_pcb *pcb) {
+    auto *self = static_cast<IpStackImpl *>(impl.get());
+    udp_bind_netif(pcb, self->netif.get());
+    udp_bind(pcb, NULL, 0);
+}
+
+void checkTimeouts() {
+    sys_check_timeouts();
+}
+
+} // namespace candy