From 0adeb8119bee59e5be076931d0187a18151e97e6 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 24 Jul 2026 10:25:12 +0200 Subject: [PATCH 1/7] chore: update AGENTS nethsecurity-api code is now inside this repo --- AGENTS.md | 1 - 1 file changed, 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index a441b31be..531b1fb3a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -199,7 +199,6 @@ Existing Python files (listed in `ruff.toml` under `extend-exclude`) are grandfa | Repo | Purpose | |---|---| -| [NethServer/nethsecurity-api](https://github.com/NethServer/nethsecurity-api) | Go API server (ns-api-server); handles JWT auth, 2FA, proxies to ubus/rpcd. **Vendored in-tree**: source lives at `packages/ns-api-server/files/src/` and is built directly from there (`Build/Prepare` copies it, no upstream fetch) — edit it in this repo, then mirror the change upstream to keep the two in sync | | [NethServer/python3-nethsec](https://github.com/NethServer/python3-nethsec) | Python utility library for APIs; used by all Python API scripts | | [NethServer/nethsecurity-ui](https://github.com/NethServer/nethsecurity-ui) | Vue 3 + Tailwind CSS management UI (standalone + controller modes); built as `ns-ui` package | | [NethServer/nethsecurity-monitoring](https://github.com/NethServer/nethsecurity-monitoring) | Go monitoring tools (ns-flows); reads netifyd flows, exposes REST API on localhost:8080 | From 7456a0f46b4ddb715309bf2e0bf8c9730c632f1d Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 24 Jul 2026 10:48:24 +0200 Subject: [PATCH 2/7] feat(python3-nethsec): move library and its tests into this repo Build the nethsec wheel from local source instead of fetching it from the old external repo, and bring its tests, test image, docs, and scans in as GitHub Actions scoped to this package. Assisted-by: Claude Code:claude-opus-4-8[1m] --- .github/workflows/python3-nethsec-docs.yml | 56 + .github/workflows/python3-nethsec-scans.yml | 49 + .../workflows/python3-nethsec-test-image.yml | 42 + .github/workflows/python3-nethsec-tests.yml | 31 + packages/python3-nethsec/.gitignore | 2 + packages/python3-nethsec/Containerfile | 33 + packages/python3-nethsec/LICENSE | 674 ++++++ packages/python3-nethsec/Makefile | 26 +- packages/python3-nethsec/README.md | 96 +- packages/python3-nethsec/builder/build.sh | 24 + packages/python3-nethsec/pytest.ini | 2 + packages/python3-nethsec/requirements.txt | 6 + packages/python3-nethsec/setup.py | 27 + .../python3-nethsec/src/nethsec/__init__.py | 3 + .../src/nethsec/conntrack/__init__.py | 137 ++ .../src/nethsec/dpi/__init__.py | 496 ++++ .../src/nethsec/firewall/__init__.py | 2146 +++++++++++++++++ .../src/nethsec/inventory/__init__.py | 950 ++++++++ .../src/nethsec/ipsec/__init__.py | 93 + .../src/nethsec/ldif/__init__.py | 418 ++++ .../src/nethsec/mwan/__init__.py | 698 ++++++ .../src/nethsec/objects/__init__.py | 884 +++++++ .../src/nethsec/ovpn/__init__.py | 285 +++ .../src/nethsec/reverse_proxy/__init__.py | 249 ++ .../src/nethsec/snort/__init__.py | 466 ++++ .../src/nethsec/users/__init__.py | 932 +++++++ .../src/nethsec/utils/__init__.py | 555 +++++ packages/python3-nethsec/test.sh | 13 + .../python3-nethsec/tests/test_conntrack.py | 455 ++++ packages/python3-nethsec/tests/test_dpi.py | 1067 ++++++++ .../python3-nethsec/tests/test_firewall.py | 1262 ++++++++++ .../python3-nethsec/tests/test_inventory.py | 1324 ++++++++++ packages/python3-nethsec/tests/test_ipsec.py | 89 + packages/python3-nethsec/tests/test_mwan.py | 610 +++++ .../python3-nethsec/tests/test_objects.py | 380 +++ packages/python3-nethsec/tests/test_ovpn.py | 63 + packages/python3-nethsec/tests/test_users.py | 521 ++++ packages/python3-nethsec/tests/test_utils.py | 229 ++ 38 files changed, 15380 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/python3-nethsec-docs.yml create mode 100644 .github/workflows/python3-nethsec-scans.yml create mode 100644 .github/workflows/python3-nethsec-test-image.yml create mode 100644 .github/workflows/python3-nethsec-tests.yml create mode 100644 packages/python3-nethsec/.gitignore create mode 100644 packages/python3-nethsec/Containerfile create mode 100644 packages/python3-nethsec/LICENSE create mode 100755 packages/python3-nethsec/builder/build.sh create mode 100644 packages/python3-nethsec/pytest.ini create mode 100644 packages/python3-nethsec/requirements.txt create mode 100644 packages/python3-nethsec/setup.py create mode 100644 packages/python3-nethsec/src/nethsec/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/conntrack/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/dpi/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/firewall/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/inventory/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/ipsec/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/ldif/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/mwan/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/objects/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/ovpn/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/reverse_proxy/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/snort/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/users/__init__.py create mode 100644 packages/python3-nethsec/src/nethsec/utils/__init__.py create mode 100755 packages/python3-nethsec/test.sh create mode 100644 packages/python3-nethsec/tests/test_conntrack.py create mode 100644 packages/python3-nethsec/tests/test_dpi.py create mode 100644 packages/python3-nethsec/tests/test_firewall.py create mode 100644 packages/python3-nethsec/tests/test_inventory.py create mode 100644 packages/python3-nethsec/tests/test_ipsec.py create mode 100644 packages/python3-nethsec/tests/test_mwan.py create mode 100644 packages/python3-nethsec/tests/test_objects.py create mode 100644 packages/python3-nethsec/tests/test_ovpn.py create mode 100644 packages/python3-nethsec/tests/test_users.py create mode 100644 packages/python3-nethsec/tests/test_utils.py diff --git a/.github/workflows/python3-nethsec-docs.yml b/.github/workflows/python3-nethsec-docs.yml new file mode 100644 index 000000000..1df0a021d --- /dev/null +++ b/.github/workflows/python3-nethsec-docs.yml @@ -0,0 +1,56 @@ +name: "python3-nethsec apidocs" + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - ".github/workflows/python3-nethsec-docs.yml" + - "packages/python3-nethsec/src/**" + pull_request: + paths: + - ".github/workflows/python3-nethsec-docs.yml" + - "packages/python3-nethsec/src/**" + +jobs: + build: + name: "Build API documentation" + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: packages/python3-nethsec + steps: + - uses: actions/checkout@v7 + - name: Set up Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install requirements for documentation generation + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install docutils pydoctor + + - name: Generate API documentation with pydoctor + run: | + pydoctor \ + --project-name=python3-nethsec \ + --project-url=https://github.com/$GITHUB_REPOSITORY \ + --html-viewsource-base=https://github.com/$GITHUB_REPOSITORY/tree/$GITHUB_SHA \ + --make-html \ + --html-output=./apidocs \ + --project-base-dir="$(pwd)" \ + --docformat=restructuredtext \ + --intersphinx=https://docs.python.org/3/objects.inv \ + ./src/nethsec + + # NOTE: this repository already publishes its GitHub Pages site from + # jekyll-gh-pages.yml, and a repository can have only one Pages source. + # So the API docs are published as a build artifact instead of a second + # (conflicting) Pages deployment. + - name: Upload API documentation artifact + uses: actions/upload-artifact@v4 + with: + name: python3-nethsec-apidocs + path: packages/python3-nethsec/apidocs diff --git a/.github/workflows/python3-nethsec-scans.yml b/.github/workflows/python3-nethsec-scans.yml new file mode 100644 index 000000000..edc29f70e --- /dev/null +++ b/.github/workflows/python3-nethsec-scans.yml @@ -0,0 +1,49 @@ +name: "python3-nethsec scans" + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - ".github/workflows/python3-nethsec-scans.yml" + - "packages/python3-nethsec/**" + pull_request: + paths: + - ".github/workflows/python3-nethsec-scans.yml" + - "packages/python3-nethsec/**" + +permissions: + contents: read + security-events: write + +jobs: + sbom: + name: SBOM + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - name: Generate SPDX SBOM + uses: aquasecurity/trivy-action@0.35.0 + with: + scan-type: fs + scan-ref: packages/python3-nethsec + format: spdx-json + output: sbom.spdx.json + - name: Generate Sarif Report + uses: aquasecurity/trivy-action@0.35.0 + with: + scan-type: fs + scan-ref: packages/python3-nethsec + format: sarif + output: trivy.sarif.json + - name: Upload report to GitHub + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: trivy.sarif.json + category: trivy-python3-nethsec + - name: Upload SBOM artifact + uses: actions/upload-artifact@v4 + with: + name: python3-nethsec-sbom + path: sbom.spdx.json diff --git a/.github/workflows/python3-nethsec-test-image.yml b/.github/workflows/python3-nethsec-test-image.yml new file mode 100644 index 000000000..7dce02d77 --- /dev/null +++ b/.github/workflows/python3-nethsec-test-image.yml @@ -0,0 +1,42 @@ +name: "python3-nethsec test image" + +on: + workflow_dispatch: + push: + paths: + - ".github/workflows/python3-nethsec-test-image.yml" + - "packages/python3-nethsec/builder/**" + - "packages/python3-nethsec/Containerfile" + - "packages/python3-nethsec/requirements.txt" + +permissions: + packages: write + +jobs: + publish_images: + name: "Build and publish" + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: packages/python3-nethsec + env: + IMAGETAG: ${{ github.ref_name }} + steps: + - uses: actions/checkout@v7 + - id: build + run: ./builder/build.sh + - id: publish + run: | + # Publish the branch + trap 'buildah logout ghcr.io' EXIT + buildah login -u ${{ github.actor }} --password-stdin ghcr.io <<<"${{ secrets.GITHUB_TOKEN }}" + images=(${{ steps.build.outputs.images }}) + urls="" + for image in "${images[@]}" ; do + buildah push $image docker://${image}:${IMAGETAG:?} + if [[ "${IMAGETAG}" == "main" || "${IMAGETAG}" == "master" ]]; then + buildah push $image docker://${image}:latest + fi + urls="${image}:${IMAGETAG} "$'\n'"${urls}" + done + echo "::notice title=Image URLs::${urls}" diff --git a/.github/workflows/python3-nethsec-tests.yml b/.github/workflows/python3-nethsec-tests.yml new file mode 100644 index 000000000..c7be82215 --- /dev/null +++ b/.github/workflows/python3-nethsec-tests.yml @@ -0,0 +1,31 @@ +name: "python3-nethsec tests" + +on: + workflow_dispatch: + push: + paths: + - ".github/workflows/python3-nethsec-tests.yml" + - "packages/python3-nethsec/**" + - "!packages/python3-nethsec/builder/**" + pull_request: + paths: + - ".github/workflows/python3-nethsec-tests.yml" + - "packages/python3-nethsec/**" + - "!packages/python3-nethsec/builder/**" + +jobs: + run_tests: + name: "Run tests" + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: packages/python3-nethsec + env: + # The test image is source-independent (source is bind-mounted at runtime). + # Use the 'main' tag on the main branch, 'latest' everywhere else — raw + # github.ref_name is '/merge' on PRs, which is not a valid image tag. + IMAGETAG: ${{ github.ref_name == 'main' && 'main' || 'latest' }} + steps: + - uses: actions/checkout@v7 + - id: test + run: ./test.sh diff --git a/packages/python3-nethsec/.gitignore b/packages/python3-nethsec/.gitignore new file mode 100644 index 000000000..59bb9ba78 --- /dev/null +++ b/packages/python3-nethsec/.gitignore @@ -0,0 +1,2 @@ +**/__pycache__/ +*.cpython* diff --git a/packages/python3-nethsec/Containerfile b/packages/python3-nethsec/Containerfile new file mode 100644 index 000000000..ed662b1f3 --- /dev/null +++ b/packages/python3-nethsec/Containerfile @@ -0,0 +1,33 @@ +FROM python:3.13.9 +RUN apt-get update \ + && apt-get install -y \ + cmake \ + liblua5.1-0-dev \ + lua5.1 \ + libjson-c-dev + +RUN mkdir /tmp/requirements \ + && git clone https://github.com/openwrt/libubox.git /tmp/requirements/libubox \ + && cd /tmp/requirements/libubox \ + && git checkout 815633847cd32ffe6da28943cbeb37edc88265c8 \ + && cmake CMakeLists.txt \ + && make install \ + && git clone https://github.com/openwrt/ubus.git /tmp/requirements/ubus \ + && cd /tmp/requirements/ubus \ + && git checkout 3cc98db1a422dcf560f2d6347fd410f17565a89d \ + && cmake CMakeLists.txt \ + && make install \ + && git clone https://github.com/openwrt/uci.git /tmp/requirements/uci \ + && cd /tmp/requirements/uci \ + && git checkout 66127cd76c5d0bd46d5a90302cc6110f53a4e2f8 \ + && cmake CMakeLists.txt \ + && make install \ + && rm -rf /tmp/requirements \ + && echo "/usr/local/lib" >> /etc/ld.so.conf.d/local.conf \ + && ldconfig + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm /tmp/requirements.txt +WORKDIR /app +CMD ["python3", "-m", "pytest"] diff --git a/packages/python3-nethsec/LICENSE b/packages/python3-nethsec/LICENSE new file mode 100644 index 000000000..f288702d2 --- /dev/null +++ b/packages/python3-nethsec/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/packages/python3-nethsec/Makefile b/packages/python3-nethsec/Makefile index 54804f55f..3181419bb 100644 --- a/packages/python3-nethsec/Makefile +++ b/packages/python3-nethsec/Makefile @@ -6,19 +6,14 @@ include $(TOPDIR)/rules.mk PKG_NAME:=python3-nethsec -# renovate: datasource=github-tags depName=NethServer/python3-nethsec PKG_VERSION:=1.7.1 PKG_RELEASE:=1 - + PKG_MAINTAINER:=Giacomo Sanchietti PKG_LICENSE:=GPL-3.0-only +PKG_LICENSE_FILES:=LICENSE -PKG_SOURCE_PROTO:=git -PKG_SOURCE_URL:=https://github.com/NethServer/python3-nethsec.git -PKG_SOURCE_VERSION:=$(PKG_VERSION) -PKG_SOURCE_SUBDIR:=python3-nethsec-$(PKG_SOURCE_VERSION) -PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) -PKG_MIRROR_HASH:=skip +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_NAME)-$(PKG_VERSION) PYTHON3_PKG_WHEEL_NAME=nethsec PYTHON3_PKG_WHEEL_VERSION:=$(PKG_VERSION) @@ -30,15 +25,26 @@ define Package/python3-nethsec SECTION:=base CATEGORY:=NethSecurity TITLE:=NethSecurity python libraries - URL:=https://github.com/NethServer/python3-nethsec/ + URL:=https://github.com/NethServer/nethsecurity/tree/master/packages/python3-nethsec DEPENDS:=+python3-uci +python3-libpass PKGARCH:=all endef - + define Package/python3-nethsec/description NethSecurity python utilities endef +# Source lives in this repository under ./src, so build the wheel from the +# local tree instead of fetching it. Only the files needed to build the wheel +# are copied; tests/ and the CI helpers are intentionally left out. +define Build/Prepare + mkdir -p $(PKG_BUILD_DIR) + $(CP) ./src $(PKG_BUILD_DIR)/ + $(CP) ./setup.py $(PKG_BUILD_DIR)/ + # setup.py reads README.md for the wheel long_description + $(CP) ./README.md $(PKG_BUILD_DIR)/ +endef + $(eval $(call Py3Package,python3-nethsec)) $(eval $(call BuildPackage,python3-nethsec)) $(eval $(call BuildPackage,python3-nethsec-src)) diff --git a/packages/python3-nethsec/README.md b/packages/python3-nethsec/README.md index 2f88b90ef..b79e1893a 100644 --- a/packages/python3-nethsec/README.md +++ b/packages/python3-nethsec/README.md @@ -1,5 +1,95 @@ -# ns-python3-nethsec +# python3-nethsec -[python3-nethsec](https://github.com/NethServer/python3-nethsec) utilities libraries. +Python3 library for NethSecurity -See online documentation: [https://nethserver.github.io/python3-nethsec/](https://nethserver.github.io/python3-nethsec/). +Requirements: + +* Python3 +* [pyuci](https://gitlab.nic.cz/turris/pyuci) + +[![Run tests](https://github.com/NethServer/nethsecurity/actions/workflows/python3-nethsec-tests.yml/badge.svg?branch=main)](https://github.com/NethServer/nethsecurity/actions/workflows/python3-nethsec-tests.yml) + +## Usage + +The `nethsec` library is composed by the following sub-packages: + +- utils +- firewall + +Usage example: +```python +from euci import EUci +from nethsec import firewall + +u = EUci() +firewall.add_to_lan(u, 'tunrw') +firewall.add_service(u, 'openvpn_rw', '1194', ['udp', 'tcp']) +firewall.apply(u) +``` + +## Documentation + +Documentation can be generated using [pydoctor](https://pydoctor.readthedocs.io) or [pydoc](https://docs.python.org/3/library/pydoc.html). + +Documentation is automatically generated at each new commit on master branch. +Online doc is hosted on [GitHub pages](https://nethserver.github.io/python3-nethsec). + +Generate doc using pydoctor: +``` +pip install -U pydoctor +pydoctor \ + --project-name=python3-nethsec \ + --project-url=https://github.com/nethserver/python3-nethsec \ + --make-html \ + --html-output=./apidocs \ + --project-base-dir="$(pwd)" \ + --docformat=restructuredtext \ + --intersphinx=https://docs.python.org/3/objects.inv \ + ./src/nethsec +``` + +Generate doc using pydoc: +```bash +cd src +python3 -m pydoc nethsec.firewall +python3 -m pydoc nethsec.utils +``` + +## Build + +Execute: +```bash +python3 -m pip install --upgrade build +python3 -m pip install wheel +python3 -m build +``` + +## Tests + +Tests run inside a podman container (see `Containerfile`) based on Python 3.13, +with the OpenWrt `libubox`/`ubus`/`uci` C libraries and the test dependencies +baked in. + +To start the tests, first make sure to have podman installed. Then, from this +package directory (`packages/python3-nethsec`) execute: +``` +./test.sh +``` + +The test container is built and published by the `python3-nethsec-test-image` +GitHub Actions workflow; `test.sh` pulls `ghcr.io/nethserver/nethsecurity/python3-nethsec-test` +(override the tag with the `IMAGETAG` environment variable, default `latest`). + +## Packaging in NethSecurity + +This library is built into the NethSecurity image as the `python3-nethsec` +OpenWrt package. The `nethsec` wheel is built from the local `./src` tree — the +source is **not** fetched from an external repository. + +When releasing a new version, bump **both** in lockstep: + +- `PKG_VERSION` (and `PKG_RELEASE`) in `Makefile` +- `version` in `setup.py` + +`PYTHON3_PKG_WHEEL_VERSION` uses `PKG_VERSION` to locate the wheel produced by +`setup.py`, so the two values must match or the install step will fail. diff --git a/packages/python3-nethsec/builder/build.sh b/packages/python3-nethsec/builder/build.sh new file mode 100755 index 000000000..d628e99b7 --- /dev/null +++ b/packages/python3-nethsec/builder/build.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# +# Create ghcr.io/nethserver/nethsecurity/python3-nethsec-test container +# + +set -e + +podman build --force-rm --layers --jobs 0 --tag ghcr.io/nethserver/nethsecurity/python3-nethsec-test . +images+=("ghcr.io/nethserver/nethsecurity/python3-nethsec-test") + +if [[ -n "${CI}" ]]; then + # Set output value for Github Actions + printf "images=%s\n" "${images[*]}" >> "${GITHUB_OUTPUT}" +else + printf "Publish the images with:\n\n" + for image in "${images[@]}"; do printf " buildah push %s docker://%s:latest\n" "${image}" "${image}" ; done + printf "\n" +fi diff --git a/packages/python3-nethsec/pytest.ini b/packages/python3-nethsec/pytest.ini new file mode 100644 index 000000000..ad5c7cc88 --- /dev/null +++ b/packages/python3-nethsec/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = . src diff --git a/packages/python3-nethsec/requirements.txt b/packages/python3-nethsec/requirements.txt new file mode 100644 index 000000000..d4d4e9c58 --- /dev/null +++ b/packages/python3-nethsec/requirements.txt @@ -0,0 +1,6 @@ +coverage==7.6.12 +pydoctor==24.11.2 +pytest==8.3.5 +pytest-mock==3.14.0 +pyuci==0.10.3 +libpass==1.9.3 diff --git a/packages/python3-nethsec/setup.py b/packages/python3-nethsec/setup.py new file mode 100644 index 000000000..638c5457f --- /dev/null +++ b/packages/python3-nethsec/setup.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages + +with open("README.md", "r") as fh: + long_description = fh.read() + +setup( + name = 'nethsec', + version = '1.7.1', + author = 'Giacomo Sanchietti', + author_email = 'giacomo.sanchietti@nethesis.it', + description = 'Utilities for NethSecurity development', + long_description = long_description, + long_description_content_type = "text/markdown", + url = "https://github.com/NethServer/python3-nethsec", + license = "GPLv3", + package_dir = {'': 'src'}, + packages = ['nethsec', 'nethsec.utils', 'nethsec.firewall', 'nethsec.mwan', 'nethsec.dpi', 'nethsec.ipsec', 'nethsec.ovpn', 'nethsec.users', 'nethsec.reverse_proxy', 'nethsec.inventory', 'nethsec.conntrack', 'nethsec.ldif', 'nethsec.objects', 'nethsec.snort'], + requires = [ "pyuci" ], + classifiers = [ + "Programming Language :: Python :: 3", + "License v3 (LGPLv3) License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Operating System :: OS Independent", + ], + python_requires = '>=3.13', +) diff --git a/packages/python3-nethsec/src/nethsec/__init__.py b/packages/python3-nethsec/src/nethsec/__init__.py new file mode 100644 index 000000000..c7c337877 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/__init__.py @@ -0,0 +1,3 @@ +''' +NethSecurity python library +''' diff --git a/packages/python3-nethsec/src/nethsec/conntrack/__init__.py b/packages/python3-nethsec/src/nethsec/conntrack/__init__.py new file mode 100644 index 000000000..8712432fb --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/conntrack/__init__.py @@ -0,0 +1,137 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Library for reading and managing network connections. +""" +import subprocess + +from xml.etree import ElementTree +from xml.etree.ElementTree import Element + + +def __parse_connection_info(flow: Element) -> dict: + """ + Parse the connection information from a flow tag. + + Args: + - flow: ElementTree.Element with the flow tag. + + Returns: + dictionary with the connection information. + """ + result = {} + # expand meta tags + for child in flow.findall('meta'): + if child.get('direction') == 'original': + layer3 = child.find('layer3') + result['source'] = layer3.find('src').text + result['destination'] = layer3.find('dst').text + layer4 = child.find('layer4') + result['protocol'] = layer4.get('protoname') + if layer4.find('sport') is not None: + result['source_port'] = layer4.find('sport').text + if layer4.find('dport') is not None: + result['destination_port'] = layer4.find('dport').text + counters = child.find('counters') + if counters is not None: + result['source_stats'] = { + 'packets': int(counters.find('packets').text), + 'bytes': int(counters.find('bytes').text) + } + if child.get('direction') == 'reply': + counters = child.find('counters') + if counters is not None: + result['destination_stats'] = { + 'packets': int(counters.find('packets').text), + 'bytes': int(counters.find('bytes').text) + } + if child.get('direction') == 'independent': + result['id'] = child.find('id').text + if child.find('timeout') is not None: + result['timeout'] = child.find('timeout').text + if child.find('unreplied') is not None: + result['unreplied'] = True + if child.find('state') is not None: + result['state'] = child.find('state').text + result['labels'] = [] + for label in child.findall('labels/label'): + result['labels'].append(label.text) + + return result + + +def list_connections(labels: list = None): + """ + List all network connections. + + Args: + - labels: optional list of label strings to filter by. Only connections that have + ALL the specified labels are returned. If ``None`` or empty, all + connections are returned. + + Returns: + list of connections. + """ + cmd = ["conntrack", "-L", "-o", "labels,xml"] + if labels: + cmd.extend(["-l", ",".join(labels)]) + result = subprocess.run(cmd, capture_output=True, text=True) + root = ElementTree.fromstring(result.stdout) + connections = [] + for flow in root.findall('flow'): + connections.append(__parse_connection_info(flow)) + + return connections + + +def drop_connection(connection_id: str): + """ + Drop a connection by its id. + + Args: + - connection_id: id of the connection to drop. + + Raises: + - ValueError: if the connection with the given id is not found. + - RuntimeError: if the connection could not be dropped. + """ + connections = list(filter(lambda x: x['id'] == connection_id, list_connections())) + if len(connections) <= 0: + raise ValueError(f"Connection with id {connection_id} not found.") + + connection = connections[0] + process_commands = [ + 'conntrack', + '-D', + '-p', + connection['protocol'], + '-s', + connection['source'], + '-d', + connection['destination'] + ] + if connection['protocol'] not in ['icmp', 'gre']: + process_commands.extend(['--sport', connection['source_port'], '--dport', connection['destination_port']]) + + try: + subprocess.run(process_commands, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Error running command: {e}") + + +def drop_all_connections(): + """ + Flush all connections. + + Raises: + - RuntimeError: if command failed to execute. + """ + try: + subprocess.run(['conntrack', '-F'], check=True, capture_output=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Error running command: {e}") diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py new file mode 100644 index 000000000..5de35668e --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -0,0 +1,496 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Library that handles the DPI rules. +""" + +import json +import subprocess +from fnmatch import fnmatch + +import math +from euci import EUci + +from nethsec import utils, firewall +from nethsec.utils import ValidationError + + +def __load_applications() -> dict[int, str]: + """ + Reads the applications from the netify-apps.conf file. + + Returns: + dict of applications, each dict contains the property "id" and "name" + """ + applications = dict[int, str]() + with open('/etc/netifyd/netify-apps.conf', 'r') as file: + for line in file.readlines(): + if line.startswith('app'): + line_split = line.strip().removesuffix('\n').removeprefix('app:').split(":") + applications[int(line_split[0])] = line_split[1] + return applications + + +def __load_application_categories() -> dict[int, dict[str]]: + """ + Reads the application categories from the netify-categories.json file. + + Returns: + dict of application categories, each dict contains the property "id" and "name" + """ + categories = dict[int, dict[str]]() + with open('/etc/netifyd/netify-categories.json', 'r') as file: + categories_file = json.load(file) + + categories_names = dict[int, str]() + if 'application_tag_index' not in categories_file: + for category_name, applications in categories_file['application_index'].items(): + for application in applications: + categories[application] = { + 'name': category_name + } + else: + categories_application_tag_index: dict[str, int] = categories_file['application_tag_index'] + for category_name, category_id in categories_application_tag_index.items(): + categories_names[category_id] = category_name + + categories_application_index: list[int, list[int]] = categories_file['application_index'] + for category_id, applications_id in categories_application_index: + for application_id in applications_id: + categories[application_id] = { + 'name': categories_names[category_id] + } + + return categories + + +def load_protocols() -> dict[int, str]: + """ + Reads the protocols from the netifyd --dump-protos command. + + Returns: + dict of protocols, each dict contains the property "id" and "name" + """ + result = subprocess.run(['netifyd', '--dump-protos'], check=True, capture_output=True) + protocols = dict[int, str]() + for line in result.stdout.decode().splitlines(): + # lines can be empty + if len(line) < 1: + continue + line_split = line.split(":") + protocols[int(line_split[0].strip())] = line_split[1].strip() + + return protocols + + +def __load_protocol_categories() -> dict[int, dict[str]]: + """ + Reads the protocol categories from the netify-categories.json file. + + Returns: + dict of protocol categories, each dict contains the property "id" and "name" + """ + categories = dict[int, dict[str]]() + with open('/etc/netifyd/netify-categories.json', 'r') as file: + categories_file = json.load(file) + + categories_names = dict[int, str]() + + if 'protocol_tag_index' not in categories_file: + for category_name, protocols in categories_file['protocol_index'].items(): + for protocol in protocols: + categories[protocol] = { + 'name': category_name + } + else: + categories_protocol_tag_index: dict[str, int] = categories_file['protocol_tag_index'] + for category_name, category_id in categories_protocol_tag_index.items(): + categories_names[category_id] = category_name + + categories_protocol_index: list[int, list[int]] = categories_file['protocol_index'] + for category_id, protocol_ids in categories_protocol_index: + for protocol_id in protocol_ids: + categories[protocol_id] = { + 'name': categories_names[category_id] + } + + return categories + + +def __load_blocklist() -> list[dict[str]]: + """ + Format the applications and protocols into a list of dicts. + + Returns: + list of dicts, each dict contains the property "id", "name", "type" and "category" + """ + result = list[dict[str]]() + applications = __load_applications() + application_categories = __load_application_categories() + + for application_id, application_name in applications.items(): + result_application = { + 'id': application_id, + 'name': application_name, + 'type': 'application' + } + if application_id in application_categories: + result_application['category'] = application_categories[application_id] + result.append(result_application) + + protocols = load_protocols() + protocol_categories = __load_protocol_categories() + + for protocol_id, protocol_name in protocols.items(): + result_protocol = { + 'id': protocol_id, + 'name': protocol_name, + 'type': 'protocol' + } + if protocol_id in protocol_categories: + result_protocol['category'] = protocol_categories[protocol_id] + result.append(result_protocol) + + return result + + +def list_devices(e_uci: EUci): + """ + List device-interface available for filtering. + + Returns: + list of dicts, each dict contains the property "interface" and "device" + """ + instance_name = list(e_uci.get('netifyd').keys())[0] + devices = e_uci.get('netifyd', instance_name, 'internal_if', default=[], list=True) + for zone in firewall.list_zones(e_uci).values(): + if zone['name'] == 'wan': + continue + network_devices = utils.get_all_devices_by_zone(e_uci, zone['name']) + devices = list(set(list(devices) + network_devices)) + ret = [] + for item in devices: + interface_name = utils.get_interface_from_device(e_uci, item) + ret.append({ + 'interface': interface_name if interface_name is not None else item, + 'device': item + }) + return ret + +def list_applications(search: str = None, limit: int = None, page: int = 1) -> dict: + """ + List applications available for filtering. + + Args: + - search: search string + - limit: limit the number of results + - page: page number + + Returns: + list of dicts, each dict contains the property "code" and "name" + """ + result = __load_blocklist() + + if search is not None: + # lower string so we can do a case-insensitive search + search = search.lower() + # I'm aware it's far from a readable code, but list comprehension is the fastest way to filter. + result = [item for item in result if + item.get('name', '').lower().find(search) != -1 or + item.get('category', {}).get('name', '').lower().find(search) != -1] + + total = len(result) + + if limit is not None: + result = result[limit * (page - 1):limit * page] + last_page = math.ceil(total / limit) + else: + last_page = 1 + + return { + 'data': result, + 'meta': { + 'last_page': last_page, + 'total': total, + } + } + + +def list_popular(e_uci: EUci, limit: int = None, page: int = 1) -> dict: + """ + List popular applications available for filtering. + + Args: + - limit: limit the number of results + - page: page number + + Returns: + list of dicts, each dict contains the property "id", "name", "type" and "category" + """ + popular_filters = e_uci.get('dpi', 'config', 'popular_filters', default=[], list=True) + block_list = {block['name']: block for block in __load_blocklist()} + result = [] + + for popular_filter in popular_filters: + if popular_filter in block_list.keys(): + result.append(block_list[popular_filter] | {'missing': False}) + else: + result.append({ + 'name': popular_filter, + 'missing': True + }) + + total = len(result) + + if limit is not None: + result = result[limit * (page - 1):limit * page] + last_page = math.ceil(total / limit) + else: + last_page = 1 + + return { + 'data': result, + 'meta': { + 'last_page': last_page, + 'total': total, + } + } + + +def list_rules(e_uci: EUci) -> list[dict[str]]: + """ + Index all rules + + Args: + - e_uci: euci instance + + Returns: + list of dicts, each dict contains the property "config-name", "description", "enabled", "interface" and "blocks" + """ + rules = list[dict[str]]() + fetch_rules = utils.get_all_by_type(e_uci, 'dpi', 'rule') + + if not fetch_rules: + return rules + + for rule_name in fetch_rules.keys(): + # skipping rules with criteria, must be custom entries + if e_uci.get('dpi', rule_name, 'criteria', default=None) is None: + # load blocklist of applications and protocols + blocklist = __load_blocklist() + # get content of rule + rule = fetch_rules[rule_name] + # prepare the data to append to rules + data_rule = dict[str]() + data_rule['config-name'] = rule_name + data_rule['enabled'] = rule.get('enabled', '1') == '1' + data_rule['device'] = rule.get('device', '*') + # from device, get the interface + interface = utils.get_interface_from_device(e_uci, data_rule['device']) + if interface is not None: + data_rule['interface'] = interface + data_rule['action'] = rule.get('action') + # get the blocked applications/protocols + data_rule['criteria'] = list[dict[str]]() + + # filter by application + application_blocklist = [item for item in blocklist if item['type'] == 'application'] + for application in rule.get('application', []): + found_app = [item for item in application_blocklist if item['name'] == application] + # there's a possibility of not finding the application due to manual edit of the config + if len(found_app) > 0: + data_rule['criteria'].append(found_app[0]) + + # filter by protocol + protocol_blocklist = [item for item in blocklist if item['type'] == 'protocol'] + for protocol in rule.get('protocol', []): + found_protocol = [item for item in protocol_blocklist if item['name'] == protocol] + # there's a possibility of not finding the protocol due to manual edit of the config + if len(found_protocol) > 0: + data_rule['criteria'].append(found_protocol[0]) + + # append rule + rules.append(data_rule) + + return rules + + +def __save_rule_data(e_uci: EUci, config_name: str, enabled: bool, device: str, action: str, applications: list[str], + protocols: list[str]): + e_uci.set('dpi', config_name, 'enabled', enabled) + e_uci.set('dpi', config_name, 'device', device) + e_uci.set('dpi', config_name, 'action', action) + e_uci.set('dpi', config_name, 'application', applications) + e_uci.set('dpi', config_name, 'protocol', protocols) + +def __save_exemption_data(e_uci: EUci, config_name: str, criteria: str, description: str, enabled: bool): + e_uci.set('dpi', config_name, 'enabled', enabled) + e_uci.set('dpi', config_name, 'criteria', criteria) + e_uci.set('dpi', config_name, 'description', description) + +def __toggle_engine(e_uci: EUci): + count_enabled = 0 + for section in e_uci.get_all('dpi'): + if e_uci.get('dpi', section, default="") == "rule" and e_uci.get('dpi', section, 'enabled', default="0") == "1": + count_enabled = count_enabled + 1 + + if count_enabled > 0: + e_uci.set('dpi', 'config', 'enabled', '1') + else: + e_uci.set('dpi', 'config', 'enabled', '0') + +def add_rule(e_uci: EUci, enabled: bool, device: str, action: str, applications: list[str], + protocols: list[str]) -> str: + """ + Store a new rule + + Args: + - e_uci: euci instance + - description: description of the rule + - enabled: enable the rule + - action: apply specific action to rule, can be 'block', 'bulk', 'best_effort', 'video' or 'voice' + - device: device to listen and apply the rule on + - applications: list of applications to block + - protocols: list of protocols to block + + Returns: + config name of the rule created + """ + rule_name = utils.get_random_id() + e_uci.set('dpi', rule_name, 'rule') + __save_rule_data(e_uci, rule_name, enabled, device, action, applications, protocols) + __toggle_engine(e_uci) + e_uci.save('dpi') + return rule_name + + +def delete_rule(e_uci: EUci, config_name: str): + """ + Delete a rule + + Args: + - e_uci: euci instance + - config_name: config name of the rule to delete + """ + e_uci.delete('dpi', config_name) + __toggle_engine(e_uci) + e_uci.save('dpi') + + +def edit_rule(e_uci: EUci, config_name: str, enabled: bool, device: str, action: str, applications: list[str], + protocols: list[str]): + """ + Edit a rule + + Args: + - e_uci: euci instance + - config_name: rule to change + - enabled: enable the rule + - device: device to listen and apply the rule on + - action: apply specific action to rule, can be 'block', 'bulk', 'best_effort', 'video' or 'voice' + - applications: array of applications to block + - protocols: array of protocols to block + + Raises + - ValidationError: if the config name is invalid + """ + if e_uci.get('dpi', config_name, default=None) is None: + raise ValidationError('config-name', 'invalid', config_name) + + __save_rule_data(e_uci, config_name, enabled, device, action, applications, protocols) + __toggle_engine(e_uci) + + e_uci.save('dpi') + +def list_exemptions(e_uci: EUci) -> list[dict[str]]: + """ + Index all global exemptions + + Args: + - e_uci: euci instance + + Returns: + list of dicts, each dict contains the property "config-name", "description", "enabled", "criteria" + """ + exemptions = list[dict[str]]() + fetch_ex = utils.get_all_by_type(e_uci, 'dpi', 'exemption') + + if not fetch_ex: + return exemptions + for ex_name in fetch_ex.keys(): + # get content of exemption + ex = fetch_ex[ex_name] + # prepare the data to append to rules + data_ex = dict[str]() + data_ex['config-name'] = ex_name + data_ex['enabled'] = ex.get('enabled', '1') == '1' + data_ex['criteria'] = ex.get('criteria', '') + data_ex['description'] = ex.get('description', '') + # append exemption + exemptions.append(data_ex) + + return exemptions + + +def add_exemption(e_uci: EUci, criteria: str, description: str, enabled: bool): + """ + Store a new global exemption + + Args: + - e_uci: euci instance + - criteria: exemption criteria, usually it's an IP address + - description: description of the rule + - enabled: enable the exemption + + Returns: + config name of the exemption created + """ + ex_list = utils.get_all_by_type(e_uci, 'dpi', 'exemption') + for ex_name in ex_list: + ex = ex_list[ex_name] + if ex.get('criteria', '') == criteria: + raise ValidationError('criteria', 'criteria_already_exists', criteria) + + ex_name = utils.get_random_id() + e_uci.set('dpi', ex_name, 'exemption') + __save_exemption_data(e_uci, ex_name, criteria, description, enabled) + e_uci.save('dpi') + return ex_name + + +def delete_exemption(e_uci: EUci, config_name: str): + """ + Delete a global exemption + + Args: + - e_uci: euci instance + - config_name: config name of the rule to delete + """ + e_uci.delete('dpi', config_name) + e_uci.save('dpi') + + +def edit_exemption(e_uci: EUci, config_name: str, criteria: str, description: str, enabled: bool): + """ + Edit a global exemption + + Args: + - e_uci: euci instance + - config_name: rule to change + - criteria: exemption criteria, usually it's an IP address + - description: description of the rule + - enabled: enable the exemption + + Raises + - ValidationError: if the config name is invalid + """ + if e_uci.get('dpi', config_name, default=None) is None: + raise ValidationError('config-name', 'invalid', config_name) + + __save_exemption_data(e_uci, config_name, criteria, description, enabled) + e_uci.save('dpi') diff --git a/packages/python3-nethsec/src/nethsec/firewall/__init__.py b/packages/python3-nethsec/src/nethsec/firewall/__init__.py new file mode 100644 index 000000000..4adb67ba5 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/firewall/__init__.py @@ -0,0 +1,2146 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +Firewall utilities +''' +import ipaddress +import json +import os +import subprocess + +from euci import EUci + +from nethsec import utils, objects + +PROTOCOLS = ['tcp', 'udp', 'udplite', 'icmp', 'esp', 'ah', 'sctp'] +TARGETS = ['ACCEPT', 'DROP', 'REJECT', 'NOTRACK'] + +def add_device_to_zone(uci, device, zone): + ''' + Add given device to a firewall zone. + The device is not added if the firewall zone does not exists + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - device -- Device name + - zone -- Firewall zone name + + Returns: + - If the firewall zone exists, the name of the section where the device has been added. + - None, otherwise. + ''' + for section in uci.get("firewall"): + s_type = uci.get("firewall", section) + if s_type == "zone": + zname = uci.get("firewall", section, "name") + if zname == zone: + try: + devices = list(uci.get_all("firewall", section, "device")) + except: + devices = [] + if not device in devices: + devices.append(device) + uci.set("firewall", section, "device", devices) + uci.save("firewall") + return section + + return None + +def add_interface_to_zone(uci, interface, zone): + ''' + Add given interface to a firewall zone. + The interface is not added if the firewall zone does not exists + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - interface -- Interface name + - zone -- Firewall zone name + + Returns: + - If the firewall zone exists, the name of the section where the device has been added. + - None, otherwise. + ''' + for section in uci.get("firewall"): + s_type = uci.get("firewall", section) + if s_type == "zone": + zname = uci.get("firewall", section, "name") + if zname == zone: + try: + networks = list(uci.get_all("firewall", section, "network")) + except: + networks = [] + if not interface in networks: + networks.append(interface) + uci.set("firewall", section, "network", networks) + uci.save("firewall") + return section + + return None + + +def remove_interface_from_zone(uci, interface, zone): + ''' + Remove the given interface from a firewall zone. + The operation always succeed if the zone does not exists + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - interface -- Interface name + - zone -- Firewall zone name + + Returns: + - If the firewall zone exists, the name of the section where the interface has been removed. + - None, otherwise. + ''' + + for z in utils.get_all_by_type(uci, 'firewall', 'zone'): + if uci.get('firewall', z, 'name') == zone: + try: + networks = list(uci.get_all("firewall", z, "network")) + except: + networks = [] + if interface in networks: + networks.remove(interface) + uci.set("firewall", z, "network", networks) + uci.save("firewall") + return z + return None + +def remove_device_from_zone(uci, device, zone): + ''' + Remove the given device from a firewall zone. + The operation always succeed if the zone does not exists + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - device -- Device name + - zone -- Firewall zone name + + Returns: + - If the firewall zone exists, the name of the section where the device has been removed. + - None, otherwise. + ''' + + for z in utils.get_all_by_type(uci, 'firewall', 'zone'): + if uci.get('firewall', z, 'name') == zone: + try: + devices = list(uci.get_all("firewall", z, "device")) + except: + devices = [] + if device in devices: + devices.remove(device) + uci.set("firewall", z, "device", devices) + uci.save("firewall") + return z + return None + +def add_device_to_lan(uci, device): + ''' + Shortuct to add a device to lan zone + + Arguments: + - uci -- EUci pointer + - device -- Device name + + Returns: + - The name of section or None + ''' + return add_device_to_zone(uci, device, 'lan') + +def add_device_to_wan(uci, device): + ''' + Shortuct to add a device to wan zone + + Arguments: + - uci -- EUci pointer + - device -- Device name + + Returns: + - The name of the configuration section or None + ''' + return add_device_to_zone(uci, device, 'wan') + +def add_vpn_interface(uci, name, device, link=""): + ''' + Create a network interface for the given device. + The interface can be used for PBR (Policy Based Routing). + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Interface name + - device -- Device name + - link -- A reference to an existing key in the format / (optional) + + Returns: + - The name of the configuration section or None in case of error + ''' + iname = utils.sanitize(name) + uci.set('network', iname, 'interface') + uci.set('network', iname, 'proto', 'none') + uci.set('network', iname, 'device', device) + uci.set('network', iname, 'ns_tag', ["automated"]) + if link: + uci.set('network', iname, 'ns_link', link) + uci.save('network') + return iname + +def add_trusted_zone(uci, name, networks = [], link = ""): + ''' + Create a trusted zone. The zone will: + - be able to access lan and wan zone + - be accessible from lan zone + + If a zone with the same name already exists, do not recreate it. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Zone name, maximum length is 12 + - network -- A list of interfaces to be added to the zone (optional) + - link -- A reference to an existing key in the format / (optional) + + Returns a tuple: + - The name of the configuration section or None in case of error + - A list of configuration sections or an empy list in case of error + ''' + + if len(name) > 12: + return None, None + + # avoid duplicated zones + zones = utils.get_all_by_type(uci, 'firewall', 'zone') + for z in zones: + if zones[z].get("name", "") == name: + return None, None + forwardings = list() + zname = utils.get_random_id() + name = utils.sanitize(name) + uci.set("firewall", zname, 'zone') + uci.set("firewall", zname, 'name', name) + uci.set("firewall", zname, 'input', 'ACCEPT') + uci.set("firewall", zname, 'output', 'ACCEPT') + uci.set("firewall", zname, 'forward', 'REJECT') + if networks: + uci.set("firewall", zname, 'network', networks) + uci.set("firewall", zname, "ns_tag", ["automated"]) + if link: + uci.set("firewall", zname, "ns_link", link) + + flan = utils.get_random_id() + uci.set("firewall", flan, "forwarding") + uci.set("firewall", flan, "src", name) + uci.set("firewall", flan, "dest", "lan") + uci.set("firewall", flan, "ns_tag", ["automated"]) + if link: + uci.set("firewall", flan, "ns_link", link) + forwardings.append(flan) + + flan = utils.get_random_id() + uci.set("firewall", flan, "forwarding") + uci.set("firewall", flan, "src", "lan") + uci.set("firewall", flan, "dest", name) + uci.set("firewall", flan, "ns_tag", ["automated"]) + if link: + uci.set("firewall", flan, "ns_link", link) + forwardings.append(flan) + + flan = utils.get_random_id() + uci.set("firewall", flan, "forwarding") + uci.set("firewall", flan, "src", name) + uci.set("firewall", flan, "dest", "wan") + uci.set("firewall", flan, "ns_tag", ["automated"]) + if link: + uci.set("firewall", flan, "ns_link", link) + forwardings.append(flan) + uci.save('firewall') + reorder_firewall_config(uci) + + return zname, forwardings + +def add_service(uci, name, port, proto, link = "", tags=None): + ''' + Create an ACCEPT traffic rule for the given service + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Service name + - port -- Service port number as string or array + - proto -- List of service protocols + - link -- A reference to an existing key in the format / (optional) + + Returns: + - The name of the configuration section + ''' + if tags is None: + tags = ['automated'] + rname = utils.get_id(f"allow_{name}") + uci.set("firewall", rname, "rule") + uci.set("firewall", rname, "name", f"Allow-{name}") + uci.set("firewall", rname, "ns_service", "custom") + uci.set("firewall", rname, "src", "wan") + uci.set("firewall", rname, "dest_port", port) + if type(proto) is str: + proto = [proto] + uci.set("firewall", rname, "proto", proto) + uci.set("firewall", rname, "target", "ACCEPT") + uci.set("firewall", rname, "enabled", "1") + uci.set("firewall", rname, "ns_tag", tags) + if link: + uci.set("firewall", rname, "ns_link", link) + uci.save('firewall') + reorder_firewall_config(uci) + + return rname + +def remove_service(uci, name): + ''' + Remove the ACCEPT traffic rule for the given service + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Service name + + Returns: + - The name of the configuration section + ''' + rname = utils.get_id(f"allow_{name}") + uci.delete("firewall", rname) + uci.save('firewall') + reorder_firewall_config(uci) + + return rname + +def disable_service(uci, name): + ''' + Disable the ACCEPT rule traffic for the given service. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Service name + + Returns: + - The name of the configuration section if found, None otherwise + ''' + rname = utils.get_id(f"allow_{name}") + try: + uci.set("firewall", rname, "enabled", "0") + except: + return None + uci.save("firewall") + return rname + +def enable_service(uci, name): + ''' + Disable the ACCEPT rule traffic for the given service + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Service name + + Returns: + - The name of the configuration section if found, None otherwise + ''' + rname = utils.get_id(f"allow_{name}") + try: + uci.set("firewall", rname, "enabled", "0") + except: + return None + uci.save("firewall") + return rname + +def apply(uci): + ''' + Apply firewall configuration: + - commit changes to firewall config + - reload the firewall service + + Arguments: + - uci -- EUci pointer + ''' + uci.commit('firewall') + subprocess.run(["/etc/init.d/firewall", "reload"], check=True) + + +def add_template_forwarding(uci, name, link = ""): + ''' + Create a forwarding from templates database. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Name of the template forwarding from the templates database + - link -- A reference to an existing key in the format / (optional) + + Returns a tuple: + - The name of the configuration section for the forwarding or None in case of error + ''' + + frecord = uci.get_all("templates", name) + fname = utils.get_random_id() + uci.set("firewall", fname, "forwarding") + for section in frecord: + uci.set("firewall", fname, section, frecord[section]) + uci.set("firewall", fname, "ns_tag", ["automated"]) + if link: + uci.set("firewall", fname, "ns_link", link) + uci.save('firewall') + reorder_firewall_config(uci) + + return fname + +def add_template_zone(uci, name, networks = [], link = ""): + ''' + Create a zone from templates database. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Name of the zone from the templates database + - network -- A list of interfaces to be added to the zone (optional) + - link -- A reference to an existing key in the format / (optional) + + Returns a tuple: + - The name of the configuration section for the zone or None in case of error + - A list of configuration section names for the forwardings or None in case of error + ''' + + dzone = uci.get_all("templates", name) + # Search for zones with the same "name" + for zone in utils.get_all_by_type(uci, "firewall", "zone"): + if uci.get("firewall", zone, "name") == dzone["name"]: + return None, None + + forwardings = list() + zname = utils.get_random_id() + forward_list = dzone.pop("forwardings", list()) + uci.set("firewall", zname, "zone") + for section in dzone: + uci.set("firewall", zname, section, dzone[section]) + if len(networks) > 0: + uci.set("firewall", zname, "network", networks) + uci.set("firewall", zname, "ns_tag", ["automated"]) + if link: + uci.set("firewall", zname, "ns_link", link) + + for forward in forward_list: + forwardings.append(add_template_forwarding(uci, forward, link)) + uci.save('firewall') + reorder_firewall_config(uci) + return (zname, forwardings) + +def add_template_service_group(uci, name, src='lan', dest='wan', link=""): + ''' + Create all rules for the given service group + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Name of the service group from the templates database + - src -- Source zone, default is 'lan'. The zone must already exists inside the firewall db + - dest -- Destination zone, default is 'wan'. The zone must already exists inside the firewall db + - link -- A reference to an existing key in the format / (optional) + + Returns: + - A list of configuration section names of each rule, None in case of error + ''' + + group = uci.get_all("templates", name) + services = group.pop("services", list()) + + if not services: + return None + + rules = dict() + sections = list() + + for service in services: + (port, proto, sdesc) = service.split("/",2) + if proto not in rules: + rules[proto] = {"ports": list(), "description": ""} + rules[proto]["ports"].append(port) + rules[proto]["description"] = utils.sanitize(sdesc) + + for proto in rules: + sname = utils.get_random_id() + desc = rules[proto]["description"] + uci.set("firewall", sname, "rule") + uci.set("firewall", sname, "name", f"Allow-{name}-{proto}") + uci.set("firewall", sname, "ns_description", f"{desc} - {proto}") + uci.set("firewall", sname, "src", src) + uci.set("firewall", sname, "dest", dest) + uci.set("firewall", sname, "proto", proto) + uci.set("firewall", sname, "dest_port", ",".join(rules[proto]["ports"])) + uci.set("firewall", sname, "target", "ACCEPT") + uci.set("firewall", sname, "enabled", "1") + uci.set("firewall", sname, "ns_tag", ["automated"]) + if link: + uci.set("firewall", sname, "ns_link", link) + sections.append(sname) + uci.save('firewall') + reorder_firewall_config(uci) + return sections + +def add_template_rule(uci, name, proto="", port="", link="", tags=None): + ''' + Create a rule from templates database. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Name of the template rule from the templates database + - proto -- A valid UCI protocol (optional) + - port -- A port or comma-separated list of ports (optional) + - link -- A reference to an existing key in the format / (optional) + + Returns: + - The name of the configuration section for the rule or None in case of error + ''' + + if tags is None: + tags = ['automated'] + drule = uci.get_all("templates", name) + rname = utils.get_random_id() + uci.set("firewall", rname, "rule") + for section in drule: + if port: + drule[section] = drule[section].replace("__PORT__", port) + if proto: + drule[section] = drule[section].replace("__PROTO__", proto) + uci.set("firewall", rname, section, drule[section]) + uci.set("firewall", rname, "ns_tag", tags) + if link: + uci.set("firewall", rname, "ns_link", link) + uci.save('firewall') + reorder_firewall_config(uci) + return rname + +def get_all_linked(uci, link): + ''' + Search all database, execpt templates one, for entities with the given link + + Arguments: + - uci -- EUci pointer + - link -- A reference to an existing key in the format / + + Returns: + - A dictionary of all matched sections like + {"db1": ["key1", "key2"], "db2": [...] } + ''' + + ret = dict() + for config in uci.list_configs(): + if config == "templates" or config.endswith(".backup"): + continue + records = utils.get_all_by_option(uci, config, 'ns_link', link, deep = False) + ret[config] = records + + return ret + +def disable_linked_rules(uci, link): + ''' + Disable all rules matching the given link + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - link -- A reference to an existing key in the format / + + Returns: + - A list of disabled sections + ''' + + disabled = list() + linked = get_all_linked(uci, link) + if "firewall" in linked: + for section in linked["firewall"]: + if uci.get("firewall", section) == "rule": + uci.set("firewall", section, "enabled", 0) + disabled.append(section) + + uci.save("firewall") + return disabled + +def delete_linked_sections(uci, link): + ''' + Delete all sections matching the given link. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - link -- A reference to an existing key in the format / + + Returns: + - A list of deleted sections + ''' + + deleted = list() + linked = get_all_linked(uci, link) + for db in linked: + for section in linked[db]: + uci.delete(db, section) + deleted.append(section) + uci.save(db) + + return deleted + +def is_ipv6_enabled(uci): + ''' + Search the network database for devices and interfaces using IPv6 + + Arguments: + - uci -- EUci pointer + + Returns: + - True if IPv6 is enabled at least on a device or interface, False otherwise + ''' + + for interface in utils.get_all_by_type(uci, 'network', 'interface'): + for option in uci.get_all('network', interface): + if option.startswith("ip6") or option == "dhcpv6" or option == "ipv6": + return True + if uci.get('network', interface, 'proto', default="") in ['6in4', '6to4', '6rd', 'grev6', 'grev6tap', 'vtiv6']: + return True + + for device in utils.get_all_by_type(uci, 'network', 'device'): + try: + if uci.get_all('network', device, 'ipv6') == 1: + return True + except: + pass + return False + +def disable_ipv6_firewall(uci): + ''' + Disable all rules, forwardings, redirects, zones and ipsets for ipv6-only family. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of disabled sections + ''' + + disabled = list() + for section_type in ["rule", "forwarding", "redirect", "zone", "ipset"]: + for section in utils.get_all_by_type(uci, 'firewall', section_type): + if uci.get("firewall", section, 'family', default="any") == "ipv6": + uci.set("firewall", section, "enabled", "0") + disabled.append(section) + + uci.save("firewall") + return disabled + + +def list_zones(uci) -> dict: + """ + Get all zones from firewall config + + Args: + uci: EUci pointer + + Returns: + dict with all zones + """ + return utils.get_all_by_type(uci, 'firewall', 'zone') + + +def list_zones_no_aliases(uci) -> dict: + """ + Get all zones from firewall config, excluding aliases in network section + + Args: + - uci: EUci pointer + + Returns: + dict with all zones + """ + zones = list_zones(uci) + for name, zone in zones.items(): + result = [] + networks = zone.get("network", []) + for network in networks: + config_network = uci.get("network", network, "device", default="", dtype=str) + if not config_network.startswith("@"): + result.append(network) + + zone["network"] = result + + return zones + + +def list_forwardings(uci) -> dict: + """ + Get all forwardings from firewall config + + Args: + uci: EUci pointer + + Returns: + dict with all forwardings + """ + return utils.get_all_by_type(uci, 'firewall', 'forwarding') + + +def add_forwarding(uci, src: str, dest: str) -> str: + """ + Add forwarding from src to dest. + + Args: + uci: EUci pointer + src: source zone, must be zone name, not config name + dest: destination zone, must be zone name, not config name + + Returns: + name of forwarding config that was added + """ + config_name = utils.get_id(f'{src}2{dest}') + uci.set('firewall', config_name, 'forwarding') + uci.set('firewall', config_name, 'src', src) + uci.set('firewall', config_name, 'dest', dest) + uci.save('firewall') + return config_name + +def get_zone_by_name(uci, name: str) -> str: + """ + Get zone config name by zone name. + + Args: + uci: EUci pointer + name: zone name + + Returns: + tuple of zone config name and zone config if zone with name name exists, (None, None) otherwise + """ + zones = utils.get_all_by_type(uci, 'firewall', 'zone') + for z in zones: + if uci.get('firewall', z, 'name', default='') == name: + return (z, zones[z]) + return (None, None) + +def get_rule_by_name(uci, name: str, tag = "") -> str: + """ + Get rule config name and rule data by rule name, optionally filtered by tag. + Assume there is only one rule with the same name. + + Args: + uci: EUci pointer + name: rule name + tag: optional tag to filter rules + + Returns: + tuple of rule config name and rule config if rule with name name exists, (None, None) otherwise + """ + rules = utils.get_all_by_type(uci, 'firewall', 'rule') + for r in rules: + if uci.get('firewall', r, 'name', default='') == name: + if not tag or tag in uci.get('firewall', r, 'ns_tag', default=[]): + return (r, rules[r]) + return (None, None) + + +def zone_exists(u, zone_name): + """ + Check if a zone with name zone_name already exists + + Args: + u: EUci pointer + zone_name: zone name to check + + Returns: + true if a zone with name zone_name already exists, false otherwise + """ + try: + for h in utils.get_all_by_type(u, 'firewall', 'zone'): + if u.get('firewall', h, 'name', default='') == zone_name: + return True + except: + return False + + return False + + +def add_zone(uci, name: str, input: str, forward: str, traffic_to_wan: bool = False, forwards_to: list[str] = None, + forwards_from: list[str] = None, log: bool = False) -> {str, set[str]}: + """ + Add zone to firewall config. + + Args: + uci: EUci pointer + name: name of zone + input: rule for input traffic, must be one of 'ACCEPT', 'REJECT', 'DROP' + forward: rule for forward traffic, must be one of 'ACCEPT', 'REJECT', 'DROP' + traffic_to_wan: if True, add forwarding from zone to wan + forwards_to: list of zones to forward traffic to + forwards_from: list of zones to forward traffic from + log: if True, log blocked traffic destined to this zone + + Returns: + tuple of zone config name and set of added forwarding configs + """ + + # check if a zone with the same name already exists + if zone_exists(uci, name): + return utils.validation_error("name", "zone_already_exists", name) + + zone_config_name = utils.get_id(name) + uci.set('firewall', zone_config_name, 'zone') + uci.set('firewall', zone_config_name, 'name', name) + uci.set('firewall', zone_config_name, 'input', input) + uci.set('firewall', zone_config_name, 'forward', forward) + uci.set('firewall', zone_config_name, 'output', 'ACCEPT') + if log: + uci.set('firewall', zone_config_name, 'log', '1') + log_opts = get_default_logging_options(uci) + uci.set('firewall', zone_config_name, 'log_limit', log_opts['zone_log_limit']) + else: + uci.set('firewall', zone_config_name, 'log', '0') + try: + uci.delete('firewall', zone_config_name, 'log_limit') + except: + pass + + + forwardings_added = set() + + if traffic_to_wan: + forwardings_added.add(add_forwarding(uci, name, 'wan')) + + if forwards_to is not None: + for forward_to in forwards_to: + forwardings_added.add(add_forwarding(uci, name, forward_to)) + + if forwards_from is not None: + for forward_from in forwards_from: + forwardings_added.add(add_forwarding(uci, forward_from, name)) + + uci.save('firewall') + reorder_firewall_config(uci) + return zone_config_name, forwardings_added + + +def edit_zone(uci, name: str, input: str, forward: str, traffic_to_wan: bool = False, forwards_to: list[str] = None, + forwards_from: list[str] = None, log: bool = False) -> {str, set[str]}: + """ + Edit an existing zone. + + Args: + uci: EUci pointer + name: name of zone to edit + input: rule for input traffic, must be one of 'ACCEPT', 'REJECT', 'DROP' + forward: rule for forward traffic, must be one of 'ACCEPT', 'REJECT', 'DROP' + traffic_to_wan: if True, add forwarding from zone to wan + forwards_to: list of zones to forward traffic to + forwards_from: list of zones to forward traffic from + log: if True, log blocked traffic destined to this zone + + Returns: + tuple of zone config name and set of updated forwarding configs + """ + (zone_config_name, zone) = get_zone_by_name(uci, name) + if zone is None or zone_config_name is None: + return utils.validation_error("name", "zone_does_not_exists", name) + + uci.set('firewall', zone_config_name, 'input', input) + uci.set('firewall', zone_config_name, 'forward', forward) + uci.set('firewall', zone_config_name, 'output', 'ACCEPT') + if log: + uci.set('firewall', zone_config_name, 'log', '1') + if uci.get('firewall', zone_config_name, 'log_limit', default=None) is None: + # set default log limit, do not overwrite existing value + log_opts = get_default_logging_options(uci) + uci.set('firewall', zone_config_name, 'log_limit', log_opts['zone_log_limit']) + else: + uci.set('firewall', zone_config_name, 'log', '0') + try: + uci.delete('firewall', zone_config_name, 'log_limit') + except: + pass + + # delete old forwardings + + forwardings = list_forwardings(uci) + to_delete_forwardings = set() + for forwarding in forwardings: + if forwardings[forwarding]['src'] == name: + to_delete_forwardings.add(forwarding) + if forwardings[forwarding]['dest'] == name: + to_delete_forwardings.add(forwarding) + + for to_delete_forwarding in to_delete_forwardings: + uci.delete('firewall', to_delete_forwarding) + + # create updated forwardings + + forwardings_added = set() + + if traffic_to_wan: + forwardings_added.add(add_forwarding(uci, name, 'wan')) + + if forwards_to is not None: + for forward_to in forwards_to: + forwardings_added.add(add_forwarding(uci, name, forward_to)) + + if forwards_from is not None: + for forward_from in forwards_from: + forwardings_added.add(add_forwarding(uci, forward_from, name)) + + uci.save('firewall') + return zone_config_name, forwardings_added + + +def delete_zone(uci, zone_config_name: str) -> {str, set[str]}: + """ + Delete zone and all forwardings that are connected to it. + + Args: + uci: EUci pointer + zone_config_name: name of zone config to delete + + Returns: + tuple of zone config name and set of deleted forwarding configs + + Raises: + ValueError: if zone_config_name is not a valid zone config name + """ + if zone_config_name not in list_zones(uci): + raise ValueError + zone_name = list_zones(uci)[zone_config_name]['name'] + forwardings = list_forwardings(uci) + to_delete_forwardings = set() + for forwarding in forwardings: + if forwardings[forwarding]['src'] == zone_name: + to_delete_forwardings.add(forwarding) + if forwardings[forwarding]['dest'] == zone_name: + to_delete_forwardings.add(forwarding) + + for to_delete_forwarding in to_delete_forwardings: + uci.delete('firewall', to_delete_forwarding) + uci.delete('firewall', zone_config_name) + uci.save('firewall') + reorder_firewall_config(uci) + return zone_config_name, to_delete_forwardings + +def add_default_ipv6_rules(uci): + """ + Add default ipv6 rules to firewall config, if they don't exist already. + + Args: + uci: EUci pointer + + Returns: + list of added rule config names + """ + ret = list() + rules = {"ip6_dhcp" : "Allow-DHCPv6", "ip6_mld" : "Allow-MLD", "ip6_icmp" : "Allow-ICMPv6-Input", "ip6_icmp_forward" : "Allow-ICMPv6-Forward"} + for r in rules: + (rule_name, rule) = get_rule_by_name(uci, rules[r], tag="automated") + if rule_name is None: + ret.append(add_template_rule(uci, r)) + return ret + +def delete_rule(uci, id: str) -> str: + """ + Delete rule from firewall config. + + Args: + uci: EUci pointer + id: name of rule config to delete + + Returns: + name of rule config that was deleted + + Raises: + ValueError: if id is not a valid rule config name + """ + if id not in list_rule_ids(uci): + raise utils.ValidationError('id', 'rule_not_found', id) + uci.delete('firewall', id) + uci.save('firewall') + reorder_firewall_config(uci) + return id + +def disable_rule(uci, id: str) -> str: + """ + Disable rule from firewall config. + + Args: + uci: EUci pointer + id: name of rule config to disable + + Returns: + name of rule config that was disabled + + Raises: + ValueError: if id is not a valid rule config name + """ + if id not in list_rule_ids(uci): + raise utils.ValidationError('id', 'rule_not_found', id) + uci.set('firewall', id, 'enabled', '0') + uci.save('firewall') + return id + +def enable_rule(uci, id: str) -> str: + """ + Enable rule from firewall config. + + Args: + uci: EUci pointer + id: name of rule config to enable + + Returns: + name of rule config that was enabled + + Raises: + ValueError: if id is not a valid rule config name + """ + if id not in list_rule_ids(uci): + raise utils.ValidationError('id', 'rule_not_found', id) + uci.set('firewall', id, 'enabled', '1') + uci.save('firewall') + return id + +def list_rule_ids (uci) -> list[str]: + """ + Get all rule ids from firewall config + + Args: + uci: EUci pointer + + Returns: + list of all rule ids + """ + return list(utils.get_all_by_type(uci, 'firewall', 'rule').keys()) + +def order_rules(uci, rule_type: str, order: list[str]) -> list[str]: + """ + Orders firewall rules, moves everything else but rules to the end of the list. + + Args: + e_uci: euci instance + rule_type: type of rule to order, must be 'input', 'output' or 'forward' + rules: which order to put rules + + Returns: + list of ordered rules entries + + Raises: + ValidationError: if a rule is not present in /etc/config/firewall + """ + rules = [] + if not rule_type in ['input', 'output', 'forward']: + raise utils.ValidationError('rule_type', 'invalid_rule_type', rule_type) + + (defaults, zones, forwardings, other_rules, forward_rules, output_rules, input_rules) = split_firewall_config(uci) + + if rule_type == 'input': + rules = input_rules + elif rule_type == 'output': + rules = output_rules + elif rule_type == 'forward': + rules = forward_rules + + for r in rules: + if r not in order: + raise utils.ValidationError('order', 'invalid_order', r) + + if rule_type == 'input': + final_order = defaults + zones + forwardings + other_rules + forward_rules + output_rules + order + elif rule_type == 'output': + final_order = defaults + zones + forwardings + other_rules + forward_rules + order + input_rules + elif rule_type == 'forward': + final_order = defaults + zones + forwardings + other_rules + order + output_rules + input_rules + + # enforce new order + index = 0 + for section in final_order: + subprocess.run(["uci", "-c", uci.confdir(), "-t", uci.savedir(), "reorder", f"firewall.{section}={index}"]) + index = index + 1 + uci.save('firewall') + + return order + +def resolve_address(uci, address: str) -> str: + """ + Resolve address to a more human-redeable name. + + Args: + uci: EUci pointer + address: address to resolve + + Returns: + resolved address as a dict with keys value, label and type + """ + for section in uci.get_all("dhcp"): + if uci.get("dhcp", section, "ip", default='') == address: + return {"value": address, "label": uci.get("dhcp", section, "name", default=address), "type": uci.get("dhcp", section)} + + for section in uci.get_all("network"): + if uci.get("network", section, "ipaddr", default='') == address: + return {"value": address, "label": section, "type": uci.get("network", section)} + if uci.get("network", section, "ip6addr", default='') == address: + return {"value": address, "label": section, "type": uci.get("network", section)} + + return {"value": address, "label": None, "type": None} + +def enrich_rule(uci, rule: dict) -> dict: + """ + Enrich rule with more human-readable data and missing fields + + Args: + uci: EUci pointer + rule: rule to enrich + + Returns: + enriched rule + """ + if 'automated' in rule.get('ns_tag', []): + rule['system_rule'] = True + else: + rule['system_rule'] = False + if not 'src_ip' in rule: + rule['src_ip'] = [] + else: + if type(rule['src_ip'] ) == tuple: + rule['src_ip'] = list(rule['src_ip']) + elif type(rule['src_ip'] ) == str: + rule['src_ip'] = [rule['src_ip']] + if not 'dest_ip' in rule: + rule['dest_ip'] = [] + else: + if type(rule['dest_ip'] ) == tuple: + rule['dest_ip'] = list(rule['dest_ip']) + elif type(rule['dest_ip'] ) == str: + rule['dest_ip'] = [rule['dest_ip']] + # try to gather info on src_ip and dest_ip + for key in ['src_ip', 'dest_ip']: + tmp = [] + for ip in rule.get(key, []): + if ip: + tmp.append(resolve_address(uci, ip)) + rule[key] = tmp + proto = rule.get('proto', "") + if not proto: + rule['proto'] = ['udp', 'tcp'] + else: + if type(proto) == tuple: + rule['proto'] = list(proto) + elif type(proto) == str: + rule['proto'] = [proto] + dest_port = rule.get('dest_port', "") + if not dest_port: + rule['dest_port'] = [] + else: + if type(dest_port) != list: + rule['dest_port'] = dest_port.split(" ") + if not 'ns_service' in rule: + if rule.get('proto') != None or rule.get('dest_port') != None: + rule['ns_service'] = 'custom' + else: + rule['ns_service'] = '' + if not 'ns_tag' in rule: + rule['ns_tag'] = [] + rule['log'] = True if rule.get('log', '0') == '1' else False + rule['enabled'] = True if rule.get('enabled', '1') == '1' else False + return rule + +def is_forward_rule(rule: dict) -> bool: + """ + Check if rule is a forward rule + + Args: + rule: rule to check + + Returns: + True if rule is a forward rule, False otherwise + """ + if rule.get('dest') and rule.get('src'): + return True + return False + +def is_input_rule(rule: dict) -> bool: + """ + Check if rule is an input rule + + Args: + rule: rule to check + + Returns: + True if rule is an input rule, False otherwise + """ + if not rule.get('dest') and rule.get('src'): + return True + return False + +def is_output_rule(rule: dict) -> bool: + """ + Check if rule is an output rule + + Args: + rule: rule to check + + Returns: + True if rule is an output rule, False otherwise + """ + if rule.get('dest') and not rule.get('src'): + return True + return False + +def is_zone(uci, name:str) -> bool: + """ + Check if name is a zone or any zone ('*') + + Args: + name: zone to check + + Returns: + True if name is a zone, False otherwise + """ + + # True zone if name is 'any' zone name -> '*' + if name == '*': + return True + + zones = list_zones(uci) + for value in zones.values(): + if 'name' in value and value['name'] == name: + return True + return False + +def list_rules(uci, rule_type = None) -> list: + """ + Get all rules from firewall config + + Args: + uci: EUci pointer + rule_type: optional rule type to filter, must be one of 'input', 'output' or 'forward' + + Returns: + a list of all rules + """ + rules = [] + i = 0 + for section in uci.get_all("firewall"): + if uci.get('firewall', section) == 'rule': + rule = uci.get_all('firewall', section) + rule['id'] = section + rule['index'] = i + if not rule_type: + rules.append(enrich_rule(uci, rule)) + else: + if rule_type =='forward' and is_forward_rule(rule): + rules.append(enrich_rule(uci, rule)) + rule['active_zone'] = is_zone(uci, rule.get('src','*')) and is_zone(uci, rule.get('dest','*')) + elif rule_type =='input' and is_input_rule(rule): + rules.append(enrich_rule(uci, rule)) + rule['active_zone'] = is_zone(uci, rule.get('src','*')) + elif rule_type =='output' and is_output_rule(rule): + rules.append(enrich_rule(uci, rule)) + rule['active_zone'] = is_zone(uci, rule.get('dest','*')) + i += 1 + return rules + +def list_forward_rules(uci) -> list: + """ + Get all forward rules from firewall config + + Args: + uci: EUci pointer + + Returns: + a list of all forward rules + """ + return list_rules(uci, 'forward') + + +def list_output_rules(uci) -> list: + """ + Get all output rules from firewall config + + Args: + uci: EUci pointer + + Returns: + a list of all output rules + """ + return list_rules(uci, 'output') + + +def list_input_rules(uci) -> list: + """ + Get all input rules from firewall config + + Args: + uci: EUci pointer + + Returns: + a list of all input rules + """ + return list_rules(uci, 'input') + + +def list_service_suggestions(): + """ + Get all services from /etc/services + + Returns: + a list of all services, each service is a dict with keys id, port, proto + """ + services = {} + if os.path.isfile('/etc/services'): + with open('/etc/services', 'r') as fp: + for line in fp: + if line.startswith('#'): + continue + try: + tmp = line.split() + name = tmp[0] + (port, proto) = tmp[1].split('/') + except Exception as e: + continue + if port not in services: + services[port] = {"id": name, "proto": [proto]} + else: + services[port]['proto'].append(proto) + else: + return [] + ret = [] + for port in services: + service = services[port] + service['port'] = int(port) + ret.append(service) + return ret + +def list_active_leases(): + """ + Get all active leases from /tmp/dhcp.leases + + Returns: + a list of all active leases, each lease is a dict with keys value, label, type + """ + ret = [] + if os.path.isfile("/tmp/dhcp.leases"): + with open("/tmp/dhcp.leases", "r") as fp: + for line in fp.readlines(): + line = line.strip() + if not line: + continue + tmp = line.split(" ") + if tmp[3] != "*": + ret.append({"value": tmp[2], "label": tmp[3], 'type': 'lease'}) + return ret + +def list_host_suggestions(uci): + """ + Get all hosts from dhcp and network config + + Args: + uci: EUci pointer + + Returns: + a list of all hosts, each host is a dict with keys value, label, type + """ + ret = [] + for section in uci.get_all("dhcp"): + if uci.get("dhcp", section, "ip", default=None): + ret.append({"value": uci.get("dhcp", section, "ip"), "label": uci.get("dhcp", section, "name"), "type": uci.get("dhcp", section)}) + for section in uci.get_all("network"): + ip = uci.get("network", section, "ipaddr", default=None) + if ip and ip != '127.0.0.1': + ret.append({"value": uci.get("network", section, "ipaddr"), "label": section, "type": "network"}) + if uci.get("network", section, "ip6addr", default=None): + ret.append({"value": uci.get("network", section, "ip6addr"), "label": section, "type": "network"}) + ret = ret + list_active_leases() + return ret + +def list_object_suggestions(uci, expand = False): + """ + Get all objects from objects config + + Args: + uci: EUci pointer + expand: if True, expand object details + + Returns: + a list of all objects, each object is a dict with keys value, label, type + """ + return objects.list_all_objects(uci, expand) + +def validate_address_format(address: str) -> bool: + """ + Validate address format. + Valid formats are: + - ip address + - ip range like 192.168.100.1-192.168.100.10 + - ip cidr + + Args: + address: address to validate + + Returns: + True if address is valid, False otherwise + """ + if not address: + return True + try: + if '/' in address: + (ip, cidr) = address.split('/') + ipaddress.ip_address(ip) + cidr = int(cidr) + if cidr < 0 or cidr > 32: + return False + elif '-' in address: + (start, end) = address.split('-') + start = ipaddress.ip_address(start) + end = ipaddress.ip_address(end) + if start > end: + return False + else: + ipaddress.ip_address(address) + except: + return False + return True + +def validate_port_format(port: str) -> bool: + """ + Validate port format. + + Args: + port: port to validate + + Returns: + True if port is valid, False otherwise + """ + if not port: + return True + try: + if '-' in port: + (start, end) = port.split('-') + start = int(start) + end = int(end) + if start < 0 or start > 65535: + return False + if end < 0 or end > 65535: + return False + if start > end: + return False + elif ',' in port: + for p in port.split(','): + p = int(p) + if p < 0 or p > 65535: + return False + else: + port = int(port) + if port < 0 or port > 65535: + return False + except: + return False + return True + +def validate_rule(uci, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, ns_src: str, ns_dst: str): + """ + Validate rule. + + Args: + src: source zone, must be zone name, not config name + src_ip: a list of source ip + dest: destination zone, must be zone name, not config name + dest_ip: a list of destination ip + proto: protocol, must be a list of protocols in "tcp", "udp", "udplite", "icmp", "esp", "ah", "sctp" + dest_port: a list of destination ports, each element cna be be a port number, a comma-separated list of port numbers or a range with `-` (eg. 80-90) + target: target, must be one of 'ACCEPT', 'REJECT', 'DROP', 'NOTRACK' + service: service name + ns_src: an object in the form `/` + ns_dst: an object in the form `/` + + Raises: + ValidationError: if rule is invalid + """ + if ns_src: + if not objects.object_exists(uci, ns_src): + raise utils.ValidationError('ns_src', 'object_not_found', ns_src) + else: # check source only if not using objects + for s in src_ip: + if not validate_address_format(s): + raise utils.ValidationError('src_ip', 'invalid_format', s) + if ns_dst: + if not objects.object_exists(uci, ns_dst): + raise utils.ValidationError('ns_dst', 'object_not_found', ns_dst) + else: # check destiation only if not using objects + for d in dest_ip: + if not validate_address_format(d): + raise utils.ValidationError('dest_ip', 'invalid_format', d) + if ns_src and ns_dst and objects.is_domain_set(uci, ns_src) and objects.is_domain_set(uci, ns_dst): + raise utils.ValidationError('ns_dst', 'domain_set_conflict', ns_dst) + if target not in TARGETS: + raise utils.ValidationError('target', 'invalid_target', target) + if service and service != '*': + if service == 'custom': + for p in proto: + if p not in PROTOCOLS: + raise utils.ValidationError('proto', 'invalid_proto', p) + for port in dest_port: + if not validate_port_format(port): + raise utils.ValidationError('dest_port', 'invalid_port', port) + else: + services = list(map(lambda x: x['id'], list_service_suggestions())) + if services and service not in services: + raise utils.ValidationError('ns_service', 'invalid_service', service) + +def get_service_by_name(name: str) -> dict: + """ + Get service by name. + + Args: + name: service name + + Returns: + service dict if service with name name exists, None otherwise + """ + for service in list_service_suggestions(): + if service['id'] == name: + return service + return None + +def setup_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, + enabled: bool = True, log: bool = False, tag = [], ns_src: str = None, ns_dst: str = None, ns_link: str = None) -> None: + """ + Set up a rule in the firewall config. + + Args: + uci: EUci pointer + id: id of the rule + name: name of the rule + src: source zone, must be zone name, not config name + src_ip: a list of source IP addresses + dest: destination zone, must be zone name, not config name + dest_ip: a list of destination IP addresses + proto: protocol, must be a list of protocols in "tcp", "udp", "udplite", "icmp", "esp", "ah", "sctp" + dest_port: a list of destination ports, each element can be a port number, a comma-separated list of port numbers, or a range with `-` (e.g., 80-90) + target: target, must be one of 'ACCEPT', 'REJECT', 'DROP', 'NOTRACK' + service: service name + enabled: if True, rule is enabled; if False, rule is disabled + log: if True, log traffic + tag: list of optional tags + ns_src: an object in the form `/` + ns_dst: an object in the form `/` + ns_link: a string in the form `/` that is used to link this rule to another entity + """ + uci.set('firewall', id, 'name', name) + uci.set('firewall', id, 'src', src) + uci.set('firewall', id, 'src_ip', src_ip) + uci.set('firewall', id, 'dest', dest) + uci.set('firewall', id, 'dest_ip', dest_ip) + + uci.set('firewall', id, 'target', target) + if service and service != '*': + if service == 'custom': + uci.set('firewall', id, 'ns_service', 'custom') + uci.set('firewall', id, 'proto', proto) + uci.set('firewall', id, 'dest_port', " ".join(dest_port)) + else: + uci.set('firewall', id, 'ns_service', service) + service = get_service_by_name(service) + uci.set('firewall', id, 'proto', service['proto']) + uci.set('firewall', id, 'dest_port', service['port']) + else: + try: + uci.set('firewall', id, 'ns_service', '*') + uci.set('firewall', id, 'proto', 'all') + uci.delete('firewall', id, 'dest_port') + except: + pass + + uci.set('firewall', id, 'enabled', '1' if enabled else '0') + if log: + uci.set('firewall', id, 'log', '1') + if uci.get('firewall', id, 'log_limit', default=None) is None: + # set default log limit, do not overwrite existing value + log_opts = get_default_logging_options(uci) + uci.set('firewall', id, 'log_limit', log_opts['rule_log_limit']) + else: + uci.set('firewall', id, 'log', '0') + try: + uci.delete('firewall', id, 'log_limit') + except: + pass + uci.set('firewall', id, 'ns_tag', tag) + if ns_src: + uci.set('firewall', id, 'ns_src', ns_src) + else: + try: + uci.delete('firewall', id, 'ns_src') + except: + pass + if ns_dst: + uci.set('firewall', id, 'ns_dst', ns_dst) + else: + try: + uci.delete('firewall', id, 'ns_dst') + except: + pass + if ns_link: + uci.set('firewall', id, 'ns_link', ns_link) + else: + try: + uci.delete('firewall', id, 'ns_link') + except: + pass + uci.save('firewall') + +def split_firewall_config(uci): + """ + Split firewall config into sections. + + Args: + uci: EUci pointer + + Returns: + tuple of lists of sections, in the following order: defaults, zones, forwardings, other_rules, forward_rules, output_rules, input_rules + """ + forward_rules = [] + output_rules = [] + input_rules = [] + other_rules = [] + zones = [] + forwardings = [] + defaults = [] + for section in uci.get_all("firewall"): + record = uci.get_all("firewall", section) + if uci.get('firewall', section) == 'rule': + if is_forward_rule(record): + forward_rules.append(section) + elif is_input_rule(record): + input_rules.append(section) + elif is_output_rule(record): + output_rules.append(section) + else: + other_rules.append(section) + elif uci.get('firewall', section) == 'zone': + zones.append(section) + elif uci.get('firewall', section) == 'forwarding': + forwardings.append(section) + else: + defaults.append(section) + + return (defaults, zones, forwardings, other_rules, forward_rules, output_rules, input_rules) + + +def reorder_firewall_config(uci): + """ + Reorder firewall config, moving all rules at the bottom. + The order in the file will be: + - defaults and includes + - zones + - forwardings + - forward rules + - output rules + - input rules + + Args: + uci: EUci pointer + """ + (defaults, zones, forwardings, other_rules, forward_rules, output_rules, input_rules) = split_firewall_config(uci) + + order = defaults + zones + forwardings + other_rules + forward_rules + output_rules + input_rules + index = 0 + for section in order: + subprocess.run(["uci", "-t", uci.savedir(), "-c", uci.confdir(), "reorder", f"firewall.{section}={index}"]) + index = index + 1 + uci.save('firewall') + +def add_rule(uci, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, + enabled: bool = True, log: bool = False, tag = [], add_to_top: bool = False, ns_src: str = None, ns_dst: str = None, ns_link: str = None) -> str: + """ + Add rule to firewall config. + + Args: + uci: EUci pointer + id: id of rule to edit + name: name of rule + src: source zone, must be zone name, not config name + src_ip: a list of source ip + dest: destination zone, must be zone name, not config name + dest_ip: a list of destination ip + proto: protocol, must be a list of protocols in "tcp", "udp", "udplite", "icmp", "esp", "ah", "sctp" + dest_port: a list of destination ports, each element cna be be a port number, a comma-separated list of port numbers or a range with `-` (eg. 80-90) + target: target, must be one of 'ACCEPT', 'REJECT', 'DROP', 'NOTRACK' + service: service name + enabled: if True, rule is enabled, if False, rule is disabled + log: if True, log traffic + tag: list of optional tags + add_to_top: if True, add rule to the top of the list, otherwise add to the bottom + ns_src: an object in the form `/` + ns_dst: an object in the form `/` + ns_link: a string in the form `/` that is used to link this rule to another entity + + Returns: + name of rule config that was added + """ + validate_rule(uci, src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) + rule = utils.get_random_id() + uci.set('firewall', rule, 'rule') + setup_rule(uci, rule, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag, ns_src, ns_dst, ns_link) + reorder_firewall_config(uci) + update_firewall_rules(uci) # expand objects and save + + if add_to_top: + rule_type = uci.get_all('firewall', rule) + if is_forward_rule(rule_type): + ids = list(map(lambda x: x['id'], list_forward_rules(uci))) + order_rules(uci, 'forward', [rule] + ids[:-1] ) + elif is_input_rule(rule_type): + ids = list(map(lambda x: x['id'], list_input_rules(uci))) + order_rules(uci, 'input', [rule] + ids[:-1] ) + elif is_output_rule(rule_type): + ids = list(map(lambda x: x['id'], list_output_rules(uci))) + order_rules(uci, 'output', [rule] + ids[:-1] ) + return rule + +def edit_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, + enabled: bool = True, log: bool = False, tag = [], ns_src: str = None, ns_dst: str = None, ns_link: str = None) -> str: + """ + Edit rule in firewall config. + + Args: + uci: EUci pointer + id: id of rule to edit + name: name of rule + src: source zone, must be zone name, not config name + src_ip: a list of source ip + dest: destination zone, must be zone name, not config name + dest_ip: a list of destination ip + proto: protocol, must be a list of protocols in "tcp", "udp", "udplite", "icmp", "esp", "ah", "sctp" + dest_port: a list of destination ports, each element cna be be a port number, a comma-separated list of port numbers or a range with `-` (eg. 80-90) + target: target, must be one of 'ACCEPT', 'REJECT', 'DROP', 'NOTRACK' + service: service name + enabled: if True, rule is enabled, if False, rule is disabled + log: if True, log traffic + tag: list of optional tags + ns_src: an object in the form `/` + ns_dst: an object in the form `/` + ns_link: a string in the form `/` that is used to link this rule to another entity + + Returns: + name of rule config that was edited + """ + if not uci.get('firewall', id, default=None): + raise utils.ValidationError("id", "rule_does_not_exists", id) + validate_rule(uci, src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) + setup_rule(uci, id, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag, ns_src, ns_dst, ns_link) + update_firewall_rules(uci) # expand objects and save + return id + +def list_nat_rules(uci) -> list: + """ + Get all nat rules from firewall config + + Args: + uci: EUci pointer + + Returns: + a list of all nat rules + """ + rules = [] + for section in uci.get_all("firewall"): + if uci.get('firewall', section) == 'nat': + rule = uci.get_all('firewall', section) + rule['id'] = section + rule.pop('proto', None) + rules.append(rule) + return rules + +def add_nat_rule(uci, name: str, target: str, src: str = '*', src_ip: str = '', dest_ip: str = '', snat_ip: str = '', device: str = '') -> str: + """ + Add nat rule to firewall config. + + Args: + uci: EUci pointer + name: name of rule + target: target, must be one of 'SNAT', 'DNAT' + src: source zone, must be zone name, not config name + src_ip: source ip + dest_ip: destination ip + snat_ip: snat ip + device: add nat rule just for specific device + + Returns: + name of rule config that was added + """ + if len(name) > 120: + raise utils.ValidationError('name', 'name_too_long', name) + if target not in ["ACCEPT", "MASQUERADE", "SNAT"]: + raise utils.ValidationError('target', 'invalid_target', target) + rule = utils.get_random_id() + uci.set('firewall', rule, 'nat') + uci.set('firewall', rule, 'name', name) + uci.set('firewall', rule, 'target', target) + uci.set('firewall', rule, 'src', src) + uci.set('firewall', rule, 'src_ip', src_ip) + uci.set('firewall', rule, 'dest_ip', dest_ip) + uci.set('firewall', rule, 'snat_ip', snat_ip) + uci.set('firewall', rule, 'proto', ["all"]) + uci.set('firewall', rule, 'device', device) + uci.save('firewall') + return rule + +def edit_nat_rule(uci, id: str, name: str, target: str, src: str = '*', src_ip: str = '', dest_ip: str = '', snat_ip: str = '', device: str = '') -> str: + """ + Edit nat rule in firewall config. + + Args: + uci: EUci pointer + id: id of rule to edit + name: name of rule + target: target, must be one of 'SNAT', 'DNAT' + src: source zone, must be zone name, not config name + src_ip: source ip + dest_ip: destination ip + snat_ip: snat ip + src_device: source device + device: add nat rule just for specific device + + Returns: + name of rule config that was edited + """ + if not uci.get('firewall', id, default=None): + raise utils.ValidationError("id", "rule_does_not_exists", id) + if len(name) > 120: + raise utils.ValidationError('name', 'name_too_long', name) + if target not in ["ACCEPT", "MASQUERADE", "SNAT"]: + raise utils.ValidationError('target', 'invalid_target', target) + uci.set('firewall', id, 'name', name) + uci.set('firewall', id, 'target', target) + uci.set('firewall', id, 'src', src) + uci.set('firewall', id, 'src_ip', src_ip) + uci.set('firewall', id, 'dest_ip', dest_ip) + uci.set('firewall', id, 'snat_ip', snat_ip) + uci.set('firewall', id, 'device', device) + uci.save('firewall') + return id + +def delete_nat_rule(uci, id: str) -> str: + """ + Delete nat rule from firewall config. + + Args: + uci: EUci pointer + id: id of rule to delete + + Returns: + name of rule config that was deleted + """ + if not uci.get('firewall', id, default=None): + raise utils.ValidationError("id", "rule_does_not_exists", id) + uci.delete('firewall', id) + uci.save('firewall') + return id + +def list_netmap_rules(uci) -> list: + """ + Get all netmap rules from netmap config + + Args: + uci: EUci pointer + + Returns: + a list of all netmap rules + """ + rules = [] + for section in uci.get_all("netmap"): + if uci.get('netmap', section) == 'rule': + rule = uci.get_all('netmap', section) + rule['id'] = section + rules.append(rule) + return rules + +def validate_netmap_rule(name: str, src: str, dest: str, map_from: str, map_to: str): + if len(name) > 120: + raise utils.ValidationError('name', 'name_too_long', name) + if src and dest: + raise utils.ValidationError('src', 'src_or_dest', src) + try: + ipaddress.ip_network(map_from) + except: + raise utils.ValidationError('map_from', 'map_from_invalid_format', map_from) + try: + ipaddress.ip_network(map_to) + except: + raise utils.ValidationError('map_to', 'map_to_invalid_format', map_to) + if src: + try: + ipaddress.ip_network(src) + except: + raise utils.ValidationError('src', 'src_invalid_format', src) + if dest: + try: + ipaddress.ip_network(dest) + except: + raise utils.ValidationError('dest', 'dest_invalid_format', dest) + + +def add_netmap_rule(uci, name: str, src: str, dest: str, device_in: list[str], device_out: list[str], map_from: str, map_to: str) -> str: + """ + Add netmap rule to netmap config. + + Args: + uci: EUci pointer + name: name of rule + src: source zone, must be zone name, not config name + dest: destination zone, must be zone name, not config name + device_in: list of incoming network interfaces + device_out: list of outgoing network interfaces + map_from: source network address + map_to: destination network address + + Returns: + name of rule config that was added + """ + validate_netmap_rule(name, src, dest, map_from, map_to) + rule = utils.get_random_id() + uci.set('netmap', rule, 'rule') + uci.set('netmap', rule, 'name', name) + uci.set('netmap', rule, 'src', src) + uci.set('netmap', rule, 'dest', dest) + uci.set('netmap', rule, 'device_in', device_in) + uci.set('netmap', rule, 'device_out', device_out) + uci.set('netmap', rule, 'map_from', map_from) + uci.set('netmap', rule, 'map_to', map_to) + uci.save('netmap') + return rule + +def edit_netmap_rule(uci, id: str, name: str, src: str, dest: str, device_in: list[str], device_out: list[str], map_from: str, map_to: str) -> str: + """ + Edit netmap rule in netmap config. + + Args: + uci: EUci pointer + id: id of rule to edit + name: name of rule + src: source zone, must be zone name, not config name + dest: destination zone, must be zone name, not config name + device_in: list of incoming network interfaces + device_out: list of outgoing network interfaces + map_from: source network address + map_to: destination network address + + Returns: + name of rule config that was edited + """ + validate_netmap_rule(name, src, dest, map_from, map_to) + uci.set('netmap', id, 'name', name) + uci.set('netmap', id, 'src', src) + uci.set('netmap', id, 'dest', dest) + uci.set('netmap', id, 'device_in', device_in) + uci.set('netmap', id, 'device_out', device_out) + uci.set('netmap', id, 'map_from', map_from) + uci.set('netmap', id, 'map_to', map_to) + uci.save('netmap') + return id + +def delete_netmap_rule(uci, id: str) -> str: + """ + Delete netmap rule from netmap config. + + Args: + uci: EUci pointer + id: id of rule to delete + + Returns: + name of rule config that was deleted + """ + if not uci.get('netmap', id, default=None): + raise utils.ValidationError("id", "rule_does_not_exists", id) + uci.delete('netmap', id) + uci.save('netmap') + return id + +def list_netmap_devices(uci) -> list: + """ + Get all network devices from ip command + + Args: + uci: EUci pointer + + Returns: + a list of all network devices + """ + devices = [] + try: + output = subprocess.run(["ip", "-j", "link", "show"], capture_output=True, text=True) + data = json.loads(output.stdout) + for device in data: + if device.get('ifname') != 'lo' and not device.get('ifname').startswith('ifb-'): + devices.append({"device": device['ifname'], "interface": utils.get_interface_from_device(uci, device['ifname'])}) + except: + pass + return devices + +# Objects + +def update_redirect_rules(uci): + """ + Update redirect rules with ipset field set and ns_src set. + + Args: + uci: EUci pointer + changed_sections: list of changed objects, each object is in the form of `/` + """ + for section in utils.get_all_by_type(uci, 'firewall', 'redirect'): + # fetch ns_src and ns_dst for each redirect rule + ns_src = uci.get('firewall', section, 'ns_src', default=None) + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + if ns_dst: + # if ns_dst is set, fetch the ip address from the object and set it in the redirect rule + # in case that the redirect is then deleted, no action is needed + ipaddr = objects.get_object_ip(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + if ns_src: + # if ns_src is set, check if it is a domain set + database, id = ns_src.split('/') + obj_type = uci.get(database, id) + if database == "objects" and obj_type == "domain": + # if it is a domain set, set the ipset field in the redirect rule + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src_net") + try: + uci.delete('firewall', f"{section}_ipset") + except: + pass + else: + # create a full ipset configuration for the redirect rule + uci.set('firewall', section, 'ipset', f"{section}_ipset") + uci.set('firewall', f"{section}_ipset", "ipset") + uci.set('firewall', f"{section}_ipset", "name", f"{section}_ipset") + uci.set('firewall', f"{section}_ipset", "match", "src_net") + uci.set('firewall', f"{section}_ipset", "enabled", "1") + uci.set('firewall', f"{section}_ipset", 'ns_link', f"firewall/{section}") + uci.set('firewall', f"{section}_ipset", "entry", objects.get_object_ips(uci, ns_src)) + + uci.save('firewall') + +def update_firewall_rules(uci): + """ + Update firewall rules with ipset field set and ns_src set. + + Args: + uci: EUci pointer + """ + for section in utils.get_all_by_type(uci, 'firewall', 'rule'): + keep_ipset = False + ns_src = uci.get('firewall', section, 'ns_src', default=None) + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + name = uci.get('firewall', section, 'name', default=None) + if ns_src: + if objects.is_domain_set(uci, ns_src): + keep_ipset = True + id = ns_src.split('/')[1] + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src") + try: + uci.delete('firewall', section, 'src_ip') + except: + pass + else: + ipaddr = objects.get_object_ips(uci, ns_src) + if ipaddr: + uci.set('firewall', section, 'src_ip', ipaddr) + + if ns_dst: + if objects.is_domain_set(uci, ns_dst): + keep_ipset = True + id = ns_dst.split('/')[1] + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} dst") + try: + uci.delete('firewall', section, 'dest_ip') + except: + pass + else: + ipaddr = objects.get_object_ips(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + + # delete ipset field if no domains are set + if not keep_ipset: + try: + uci.delete('firewall', section, 'ipset') + except: + pass + uci.save('firewall') + +def list_object_suggestions(uci, expand = False): + """ + Get all objects from objects config + + Args: + uci: EUci pointer + expand: if True, expand object details + + Returns: + a list of all objects, each object is a dict with keys value, label, type + """ + return objects.list_all_objects(uci, expand) + +def get_default_logging_options(uci): + """ + Get default logging options + + Args: + uci: EUci pointer + + Returns: + a dict with keys rule_log_limit, zone_log_limit, redirect_log_limit + """ + + ret = {} + defaults = uci.get('firewall', 'ns_defaults', 'rule_log_limit', default=None) + if defaults is None: + # Set defaults, if needed + uci.set('firewall', 'ns_defaults', 'defaults') + uci.set('firewall', 'ns_defaults', 'rule_log_limit', '1/s') + uci.set('firewall', 'ns_defaults', 'zone_log_limit', '5/s') + uci.set('firewall', 'ns_defaults', 'redirect_log_limit', '1/s') + uci.save('firewall') + + for key in ['rule_log_limit', 'zone_log_limit', 'redirect_log_limit']: + ret[key] = uci.get('firewall', 'ns_defaults', key) + + return ret + +def apply_default_logging_options(uci): + """ + Walk all rules, zones and redirect rules and set logging options to default + + Args: + uci: EUci pointer + + Returns: + None + """ + log_opts = get_default_logging_options(uci) + for section in uci.get_all("firewall"): + if uci.get('firewall', section) == 'rule' and uci.get('firewall', section, 'log', default='0') == '1': + uci.set('firewall', section, 'log_limit', log_opts['rule_log_limit']) + elif uci.get('firewall', section) == 'zone' and uci.get('firewall', section, 'log', default='0') == '1': + uci.set('firewall', section, 'log_limit', log_opts['zone_log_limit']) + elif uci.get('firewall', section) == 'redirect' and uci.get('firewall', section, 'log', default='0') == '1': + uci.set('firewall', section, 'log_limit', log_opts['redirect_log_limit']) + uci.save('firewall') \ No newline at end of file diff --git a/packages/python3-nethsec/src/nethsec/inventory/__init__.py b/packages/python3-nethsec/src/nethsec/inventory/__init__.py new file mode 100644 index 000000000..079a680f2 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/inventory/__init__.py @@ -0,0 +1,950 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + + +from math import floor +from euci import EUci +from nethsec import utils, mwan, users, firewall, objects +import os +import re +import csv +import subprocess +import configparser +import json +import hashlib +from socket import inet_ntoa +from struct import pack + +## Utilities + +def _run(cmd): + try: + proc = subprocess.run(cmd, shell=True, check=True, capture_output=True, text=True) + return proc.stdout.rstrip().lstrip() + except: + return '' + +def _run_status(cmd): + try: + proc = subprocess.run(cmd, shell=True, check=True, capture_output=True, text=True) + return proc.returncode + except: + return 1 + +def _run_json(cmd): + try: + return json.loads(_run(cmd)) + except: + return {} + +def _get_role(uci: EUci, interface): + for zone in utils.get_all_by_type(uci, 'firewall', 'zone'): + name = uci.get('firewall', zone, 'name') + networks = uci.get('firewall', zone, 'network', list=True, default=[]) + if interface in networks: + if name == "lan": + return "green" + elif name == "wan": + return "red" + else: + return name + +def _get_ip(interface, uci: EUci): + info = _run_json(f"ifstatus {interface}") + try: + return anonymize(info['ipv4-address'][0]['address'], uci) + except: + return '' + +def _get_mask(interface): + info = _run_json(f"ifstatus {interface}") + try: + m = info['ipv4-address'][0]['mask'] + bits = 0xffffffff ^ (1 << 32 - int(m)) - 1 + mask = inet_ntoa(pack(">I", bits)) + return mask + except: + return '' + +def _get_gateway(interface, uci: EUci): + info = _run_json(f"ifstatus {interface}") + try: + for r in info['route']: + if r['target'] == '0.0.0.0': + return anonymize(r['nexthop'], uci) + except: + return '' + +def _get_cpu_field(field, cpu_info): + for f in cpu_info: + if f['field'].startswith(field): + return f['data'] + + return '' + +def anonymize(value, uci: EUci): + if fact_subscription_status(uci).get('status', 'no') != "no": + return value + h = hashlib.sha256(value.encode()).hexdigest() + return f"anon-{h[:16]}" + +def parse_version(version_str): + # Remove "NethSecurity " prefix if present + if version_str.startswith('NethSecurity '): + version_str = version_str[13:] # len('NethSecurity ') = 13 + # Take only the part before "-" + version_str = version_str.split('-')[0] + # Convert to tuple of integers for comparison + try: + return tuple(int(x) for x in version_str.split('.')) + except (ValueError, AttributeError): + return () + +def get_networks(uci: EUci): + networks = {} + devices = utils.get_all_by_type(uci, 'network', 'device') + for section in utils.get_all_by_type(uci, 'network', 'interface'): + if section == "lan6" or section == "wan6": # skip IPv6 for now + continue + interface = uci.get_all('network', section) + device = interface.get("device") + if not device or device == "lo" or device.startswith("ipsec") or device.startswith("tun"): + continue + network = {"type": "ethernet", "name": device, "props": { "role": _get_role(uci, section), "ipaddr": _get_ip(section, uci), "netmask": _get_mask(section), "gateway": _get_gateway(section, uci)}} + # get bridge ports, exclude vlans over bridges + is_vlan = bool(re.search(r'\.\d+$', interface['device'])) # check if the device name ends with . + if interface['device'].startswith('br') and not is_vlan: + network["type"] = "bridge" + for d in devices: + if uci.get('network', d, 'name') == device: + network['props']["bridge"] = uci.get('network', d, 'ports', default=[]) + networks[device] = network + return networks + +def get_version(): + version = "" + with open('/etc/os-release', 'r') as file: + for line in file: + if line.startswith("VERSION_ID="): + version = line.split('=')[1].replace('"','').rstrip() + break + return version + +def get_product(): + product = "" + try: + with open('/sys/devices/virtual/dmi/id/product_name', 'r') as f: + product = f.read().strip() + except: + pass + if not product: + try: + with open('/etc/board.json', 'r') as f: + binfo = json.load(f) + product = binfo['model']['name'] + except: + product = "" + return product + +def is_virtual(): + cpu_info = _run_json('lscpu -J')['lscpu'] + return _get_cpu_field("Hypervisor vendor", cpu_info) if _get_cpu_field("Hypervisor vendor", cpu_info) else 'physical' + +def get_cpu_info(): + cpu_info = _run_json('lscpu -J')['lscpu'] + return { + "model": _get_cpu_field("Model name", cpu_info), + "architecture": _get_cpu_field("Architecture", cpu_info) + } + +def get_pci_info(): + # map kernel driver to device id + drivers = {} + for line in _run("find /sys | grep '.*/drivers/.*/0000:.*$' | cut -d'/' -f6,7").split('\n'): + try: + (driver,bus) = line.split("/0000:") + drivers[bus] = driver + except: + continue + + # lspci -n: 00:1b.0 0403: 8086:293e (rev 03) + # fields: bus class vendor:device revision + pci = {} + if os.path.isdir('/proc/bus/pci'): + for line in _run("lspci -n").split("\n"): + revision = '' + fields = line.split(" ", maxsplit=4) + (vendor, device) = fields[2].split(":") + if len(fields) > 3: + revision = fields[4] + pci[fields[0]] = {"class_id": fields[1].rstrip(":"), "vendor_id": vendor, "device_id": device, "revision": revision.strip(')')} + + # lspci -mm: 00:00.0 "Host bridge" "Intel Corporation" "82G33/G31/P35/P31 Express DRAM Controller" -p00 "Red Hat, Inc." "QEMU Virtual Machine" + for fields in csv.reader(_run("lspci -mm").split("\n"), delimiter=' ', quotechar='"'): + pci[fields[0]]['class_name'] = fields[1].strip('"') + pci[fields[0]]['vendor_name'] = fields[2] + pci[fields[0]]['device_name'] = fields[3] + pci[fields[0]]['driver'] = drivers.get(fields[0], '') + return pci + +def get_memory(): + result = {} + try: + with open('/proc/meminfo', 'r') as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split(':') + if len(parts) == 2: + key = parts[0].strip() + value = parts[1].strip().split()[0] # Get the number, skip the unit (kB) + result[key] = int(value) * 1024 # Convert from kB to bytes + except: + pass + if result: + result["MemUsed"] = result.get('MemTotal', 0) - result.get('MemFree', 0) - result.get('Buffers', 0) - result.get('Cached', 0) + result["SwapUsed"] = result.get('SwapTotal', 0) - result.get('SwapFree', 0) + return result + +def get_mount_points(): + result = {} + exclude_mounts = {'/rom', '/overlay', '/dev'} + seen_mounts = set() + + try: + output = _run("df -P") + lines = output.strip().split('\n') + # Skip header line + for line in lines[1:]: + if not line.strip(): + continue + parts = line.split() + if len(parts) >= 6: + mount_point = parts[5] + # Skip excluded mount points + if mount_point in exclude_mounts: + continue + # Skip duplicates (keep first occurrence) + if mount_point in seen_mounts: + continue + try: + total = int(parts[1]) * 1024 # Convert from 1K blocks to bytes + used = int(parts[2]) * 1024 + available = int(parts[3]) * 1024 + result[mount_point] = { + 'total_bytes': total, + 'used_bytes': used, + 'available_bytes': available + } + seen_mounts.add(mount_point) + except (ValueError, IndexError): + continue + except: + pass + + return result + +## Facts +## These functions are used to send info about system to legacy my.nethesis.it +## The format should not be changed until legacy my.nethesis.it is alive + +def fact_hotspot(uci: EUci): + enabled = uci.get('dedalo', 'config', 'disabled', default='1') == '0' + server = uci.get('dedalo', 'config', 'api_url', default='') + interface = uci.get('dedalo', 'config', 'interface', default='') + return { 'enabled': enabled, 'server': server, 'interface': interface } + +def fact_netifyd(uci: EUci): + try: + config = configparser.ConfigParser() + config.read('/etc/netifyd.conf') + enable_sink = config.get('netifyd', 'enable_sink') + except: + enable_sink = 'no' + return { 'enabled': enable_sink == 'yes' } + +def fact_flashstart(uci: EUci): + enabled = uci.get('flashstart', 'global', 'enabled', default='0') == '1' + bypass = len(uci.get('flashstart', 'global', 'bypass', list=True, default=[])) + pro_plus = uci.get('flashstart', 'global', 'proplus', dtype=bool, default=False) + custom_servers = len(uci.get('flashstart', 'global', 'custom_servers', list=True, default=[])) + return { 'enabled': enabled, 'bypass': bypass, 'pro_plus': pro_plus, 'custom_servers': custom_servers } + +def fact_openvpn_rw(uci: EUci): + ret = { 'enabled': 0, 'server': 0, 'instances': [] } + for section in utils.get_all_by_type(uci, 'openvpn', 'openvpn'): + if uci.get("openvpn", section, 'ns_auth_mode', default=''): + ret["server"] += 1 + if uci.get("openvpn", section, 'enabled', default='0') == '1': + ret["enabled"] += 1 + for section in utils.get_all_by_type(uci, 'openvpn', 'openvpn'): + vpn = uci.get_all("openvpn", section) + if not section.startswith('ns_'): + continue + if "ns_auth_mode" in vpn: + # we are in a ovpn_rw + instance = { + 'section': section, + 'authentication': vpn.get('ns_auth_mode'), + 'user_database': vpn.get('ns_user_db'), + 'mode': vpn.get('dev_type') + } + ret['instances'].append(instance) + return ret + +def fact_openvpn_tun(uci: EUci): + ret = { 'client': 0, 'server': 0, 'tunnels': [] } + for section in utils.get_all_by_type(uci, 'openvpn', 'openvpn'): + vpn = uci.get_all("openvpn", section) + if 'ns_auth_mode' in vpn or not section.startswith('ns_'): + continue + if vpn.get("client", "0") == "1" or vpn.get("ns_client", "0") == "1": + ret["client"] += 1 + else: + ret["server"] += 1 + instance = { + 'section': section, + 'mode': vpn.get('dev_type') + } + ret['tunnels'].append(instance) + return ret + +def fact_certificates_info(uci: EUci): + result = { + "custom_certificates": { + "count": 0 + }, + "acme_certificates": { + "count": 0, + "issued": 0, + "pending": 0 + } + } + + # Count custom certificates + try: + for entry in os.scandir('/etc/nginx/custom_certs'): + if entry.is_file() and entry.name.endswith('.crt') and os.path.isfile(entry.path[:-4] + '.key'): + result["custom_certificates"]["count"] += 1 + except Exception as e: + # Handle exceptions appropriately + pass + + # Count ACME certificates + try: + requested_certificates = utils.get_all_by_type(uci, 'acme', 'cert') + enabled_certificates = [certificate for certificate in requested_certificates + if requested_certificates[certificate]['enabled'] == '1'] + for certificate in enabled_certificates: + result["acme_certificates"]["count"] += 1 + domain = requested_certificates[certificate]['domains'][0] + cert_path = f'/etc/ssl/acme/{domain}.fullchain.crt' + if os.path.isfile(cert_path): + result["acme_certificates"]["issued"] += 1 + else: + result["acme_certificates"]["pending"] += 1 + except Exception as e: + # Handle exceptions appropriately + pass + + return result + +def fact_subscription_status(uci: EUci): + return { 'status': uci.get('ns-plug', 'config', 'type', default='no') } + +def fact_controller(uci: EUci): + if uci.get('ns-plug', 'config', 'server', default='') and uci.get('ns-plug', 'config', 'unit_id', default='') and uci.get('ns-plug', 'config', 'token', default=''): + return { "enabled": True} + else: + return { "enabled": False} + +def fact_rpcd_users(uci: EUci): + count = 0 + for section, options in (utils.get_all_by_type(uci, 'rpcd', 'login') or {}).items(): + # skip root and the controller user (controller has a random username) + if options.get('username') == 'root' or section == 'controller': + continue + count += 1 + return { 'count': count } + +def fact_threat_shield(uci: EUci): + ret = { 'enabled': False, 'community': 0, 'enterprise': 0 } + ret['enabled'] = uci.get('banip', 'global', 'ban_enabled', default='0') == '1' + try: + for feed in uci.get_all("banip", "global", "ban_feed"): + if feed.startswith("nethesis") or feed.startswith("yoroi"): + ret['enterprise'] += 1 + else: + ret['community'] += 1 + except: + pass + return ret + +def fact_adblock(uci: EUci): + ret = { 'enabled': False, 'community': 0, 'enterprise': 0 } + ret['enabled'] = uci.get('adblock', 'global', 'ts_enabled', default='0') == '1' + try: + enabled_feeds = list(uci.get_all('adblock', 'global', 'adb_sources')) + except: + enabled_feeds = [] + for feed in enabled_feeds: + if feed.startswith("nethesis") or feed.startswith("yoroi"): + ret['enterprise'] += 1 + else: + ret['community'] += 1 + return ret + +def fact_ui(uci: EUci): + ret = { 'luci': False, 'port443': False, 'port9090': False } + ret['luci'] = uci.get('ns-ui', 'config', 'luci_enable', default='0') == '1' + ret['port443'] = uci.get('ns-ui', 'config', 'nsui_enable', default='0') == '1' + ret['port9090'] = uci.get('ns-ui', 'config', 'nsui_extra_enable', default='0') == '1' and uci.get('ns-ui', 'config', 'nsui_extra_port', default='0') == '9090' + return ret + +def fact_network(uci: EUci): + result: dict = { + "zones": [] + } + vlan_count = 0 + bridge_count = 0 + bond_count = 0 + zone_network_counts = {} + route_info = { + "count_ipv6_route": 0, + "count_ipv4_route": 0 + } + # Regex pattern to match interfaces that end with "." + vlan_pattern = re.compile(r'\.\d+$') + interfaces = utils.get_all_by_type(uci, 'network', 'interface') + + # Loop through all firewall zones to gather network information + for zone in utils.get_all_by_type(uci, 'firewall', 'zone').values(): + zone_info = { + 'name': zone['name'], + 'ipv4': 0, + 'ipv6': 0 + } + devices = utils.get_all_devices_by_zone(uci, zone['name'], True) + for device in devices: + interface = utils.get_interface_from_device(uci, device) + if interface is None: + continue + is_ipv6 = False + for option in uci.get_all('network', interface): + if option.startswith("ip6") or option == "dhcpv6" or option == "ipv6": + is_ipv6 = True + break + if uci.get('network', interface, 'proto', default="") in ['dhcpv6', '6in4', '6to4', '6rd', 'grev6', 'grev6tap', 'vtiv6']: + is_ipv6 = True + if is_ipv6: + zone_info['ipv6'] += 1 + else: + zone_info['ipv4'] += 1 + # Count VLAN, bridge, and bond interfaces + if 'device' in interfaces.get(interface, {}): + device_name = interfaces[interface]['device'] + if vlan_pattern.search(device_name): + vlan_count += 1 + if device_name.startswith('br-'): + bridge_count += 1 + if device_name.startswith('bond-'): + bond_count += 1 + result["zones"].append(zone_info) + # Count networks for each zone + networks = uci.get('firewall', 'ns_'+zone['name'], 'network', list=True, default=[]) + network_count = len(networks) + # Count devices for each zone (if networks are not defined, hotspot zones, openvpn zones, etc.) + devices = utils.get_all_devices_by_zone(uci, zone['name'], True) + # remove tun-dedalo if present (it's a virtual device, we count the real interface + devices = [d for d in devices if not d.startswith('tun-dedalo')] + devices_count = len(devices) + zone_network_counts[zone['name']] = network_count or devices_count # Use the number of networks if available, otherwise use the number of devices + # Get route information + routes_ipv6 = utils.get_all_by_type(uci, 'network', 'route6') + for _ in routes_ipv6: + route_info["count_ipv6_route"] += 1 + + routes_ipv4 = utils.get_all_by_type(uci, 'network', 'route') + for _ in routes_ipv4: + route_info["count_ipv4_route"] += 1 + + # Add VLAN, bridge, and bond counts to the result + result['interface_counts'] = { + 'vlans': vlan_count, + 'bridges': bridge_count, + 'bonds': bond_count + } + # Add network zone counts to the result + result['zone_network_counts'] = zone_network_counts + + # Add route information to the result + result['route_info'] = route_info + + result['configuration'] = get_networks(uci) + + return result + +def fact_database_stats(uci: EUci): + ret = {} + databases = users.list_databases(uci) + for db in databases: + name = db["name"] + number_users = len(users.list_users(uci, name)) + ret[name] = { "users": number_users } + return ret + +def fact_firewall_stats(uci: EUci): + result = { + "firewall": { + "port_forward": 0, + "nat": {"masquerade": 0, "snat": 0, "accept": 0}, + "netmap": {"source": 0, "destination": 0}, + "rules": {"forward": 0, "input": 0, "output": 0} + }, + "objects": { + "domains": 0, + "hosts": 0, + "port_forward": {"allowed_from": 0, "destination_to": 0}, + "mwan_rules": 0, + "rules": {"forward": 0, "input": 0, "output": 0} + } + } + + # Firewall Information + # Count port forward + result["firewall"]["port_forward"] = len(utils.get_all_by_type(uci, 'firewall', 'redirect')) + + # Count NAT rules + for rule in firewall.list_nat_rules(uci): + if rule['target'] == 'ACCEPT': + result["firewall"]["nat"]["accept"] += 1 + elif rule['target'] == 'MASQUERADE': + result["firewall"]["nat"]["masquerade"] += 1 + elif rule['target'] == 'SNAT': + result["firewall"]["nat"]["snat"] += 1 + + # Count netmap rules + for rule in firewall.list_netmap_rules(uci): + if rule.get('dest', ''): + result["firewall"]["netmap"]["source"] += 1 + elif rule.get('src', ''): + result["firewall"]["netmap"]["destination"] += 1 + + # Count rules + result["firewall"]["rules"]["forward"] = len(firewall.list_forward_rules(uci)) + result["firewall"]["rules"]["input"] = len(firewall.list_input_rules(uci)) + result["firewall"]["rules"]["output"] = len(firewall.list_output_rules(uci)) + + # Object Information + # Count objects + result["objects"]["domains"] = len(objects.list_domain_sets(uci)) + result["objects"]["hosts"] = len(objects.list_host_sets(uci)) + + # Count object for port forward + for key, value in utils.get_all_by_type(uci, 'firewall', 'redirect').items(): + if isinstance(value.get('ns_src'), str): + result["objects"]["port_forward"]["allowed_from"] += 1 + if isinstance(value.get('ns_dst'), str): + result["objects"]["port_forward"]["destination_to"] += 1 + + # Count object for rules + for value in firewall.list_forward_rules(uci): + if isinstance(value.get('ns_dst'), str): + result["objects"]["rules"]["forward"] += 1 + for value in firewall.list_input_rules(uci): + if isinstance(value.get('ns_src'), str): + result["objects"]["rules"]["input"] += 1 + for value in firewall.list_output_rules(uci): + if isinstance(value.get('ns_dst'), str): + result["objects"]["rules"]["output"] += 1 + for value in mwan.index_rules(uci): + if isinstance(value.get('ns_src'), str): + result["objects"]["mwan_rules"] += 1 + + return result + +def fact_storage(uci: EUci): + return {"enabled": uci.get("fstab", "ns_data", "enabled", default="0") == "1"} + +def fact_clm(uci: EUci): + return {"enabled": uci.get("ns-clm", "config", "enabled", default="0") == "1"} + +def fact_proxy_pass(uci: EUci): + ret = { "count": 0} + try: + for l in utils.get_all_by_type(uci, 'nginx', 'location'): + if uci.get('nginx', l, 'proxy_pass', default=''): + ret["count"] += 1 + except: + pass + return ret + +def fact_dpi(uci: EUci): + ret = {"enabled": False, "rules": 0} + ret["enabled"] = uci.get('dpi', 'config', 'enabled', default='0') == '1' + for rule in utils.get_all_by_type(uci, 'dpi', 'rule'): + if uci.get('dpi', rule, 'enabled', default='0') == '1': + ret["rules"] += 1 + return ret + +def fact_dhcp_server(uci: EUci): + result = { + 'count': 0, # Initialize the count for DHCP servers + 'static_leases': 0, + 'dynamic_leases': 0, + 'dns_records_count': 0, + 'dns_forwarder_enabled': False + } + + # Count DHCP servers + for section in utils.get_all_by_type(uci, 'dhcp', 'dhcp'): + if uci.get('dhcp', section, 'dhcpv4', default='') == 'server' or uci.get('dhcp', section, 'dhcpv6', default='') == 'server': + result['count'] += 1 # Increment the count for DHCP servers + + # Count static leases + result['static_leases'] = len(utils.get_all_by_type(uci, 'dhcp', 'host')) + + static_leases = [] + for l in utils.get_all_by_type(uci, 'dhcp', 'host'): + ldata = uci.get_all('dhcp', l) + if 'mac' in ldata and 'ip' in ldata: + static_leases.append(ldata['mac'].lower()) + + # Count dynamic leases, skipping static leases + try: + with open("/tmp/dhcp.leases", "r") as fp: + for line in fp.readlines(): + tmp = line.split(" ") + if tmp[1].lower() not in static_leases: + result['dynamic_leases'] += 1 + except FileNotFoundError: + # Handle the case where the leases file doesn't exist + pass + + # Count DNS records and check if DNS forwarder is enabled + for section in utils.get_all_by_type(uci, 'dhcp', 'dnsmasq'): + servers = uci.get('dhcp', section, 'server', default=[]) + if servers: + result['dns_forwarder_enabled'] = True + for r in utils.get_all_by_type(uci, 'dhcp', 'domain'): + result['dns_records_count'] += 1 + + return result + +def fact_multiwan(uci: EUci): + policies = mwan.index_policies(uci) + result = { + 'enabled': len(policies) > 0, + 'policies': { + 'backup': 0, + 'balance': 0, + 'custom': 0 + }, + 'rules': len(mwan.index_rules(uci)) + } + for policy in policies: + result['policies'][policy['type']] += 1 + return result + +def fact_qos(uci: EUci): + ret = { + "count": 0, + "rules": [] + } + for key, interface in utils.get_all_by_type(uci, 'qosify', 'interface').items(): + if interface['disabled'] == '0': + ret["count"] += 1 + rule = { + 'enabled': interface['disabled'] == '0', + 'upload': int(interface['bandwidth_up'].removesuffix('mbit')), + 'download': int(interface['bandwidth_down'].removesuffix('mbit')), + } + ret['rules'].append(rule) + return ret + +def fact_ipsec(uci: EUci): + try: + count = len(utils.get_all_by_type(uci, 'ipsec', 'remote')) + except: + count = -1 + return { 'count': count } + +def fact_nathelpers(uci: EUci): + # count the number of lines in the file + try: + with open('/etc/modules.d/ns-nathelpers') as f: + count = len(f.readlines()) + except: + count = 0 + return { 'count': count, 'enabled': count > 0 } + +def fact_ddns(uci: EUci): + ddns = _run_status("/etc/init.d/ddns enabled") + return { 'enabled': ddns == 0 } + +def fact_snmp (uci: EUci): + snmp = _run_status("/etc/init.d/snmpd running") + return { 'enabled': snmp == 0 } + +def fact_wireguard(uci: EUci): + servers = dict() + interfaces = utils.get_all_by_type(uci, "network", "interface") + for i in interfaces: + if interfaces[i].get("proto", "") != "wireguard": + continue + peers = utils.get_all_by_type(uci, 'network', f'wireguard_{i}') + routing_all_traffic = 0 + for peer in peers: + if uci.get('network', peer, 'ns_route_all_traffic', dtype=bool, default=False): + routing_all_traffic += 1 + servers[i] = { + "peers": len(peers), + "routing_all_traffic": routing_all_traffic + } + + return { + "enabled": len(servers) > 0, + "servers": servers + } + + +def fact_snort(uci: EUci): + ret = { 'enabled': False, 'policy': '', 'oink_enabled': False, 'disabled_rules': 0, 'suppressed_rules': 0, 'bypass_src_ipv4': 0, 'bypass_src_ipv6': 0, 'bypass_dst_ipv4': 0, 'bypass_dst_ipv6': 0 } + + ret['enabled'] = uci.get('snort', 'snort', 'enabled', dtype=bool, default=False) + ret['policy'] = uci.get('snort', 'snort', 'ns_policy', default='') + ret['oink_enabled'] = True if uci.get('snort', 'snort', 'oinkcode', default='') else False + + # count list of ns_disabled_rules + ret['disabled_rules'] = len(uci.get('snort', 'snort', 'ns_disabled_rules', list=True, default=[])) + # count list of ns_suppress rules + ret['suppressed_rules'] = len(uci.get('snort', 'snort', 'ns_suppress', list=True, default=[])) + # count the source bypass of ipv4 and ipv6 + ret['bypass_src_ipv4'] = len(uci.get('snort', 'nfq', 'bypass_src_v4', list=True, default=[])) + ret['bypass_src_ipv6'] = len(uci.get('snort', 'nfq', 'bypass_src_v6', list=True, default=[])) + ## count the destination bypass_dst_v4 and bypass_dst_ipv6 + ret['bypass_dst_ipv4'] = len(uci.get('snort', 'nfq', 'bypass_dst_v4', list=True, default=[])) + ret['bypass_dst_ipv6'] = len(uci.get('snort', 'nfq', 'bypass_dst_v6', list=True, default=[])) + + return ret + +def fact_mac_ip_binding(uci: EUci): + ret = { "disabled": 0, "soft-binding": 0, "hard-binding": 0 } + # Parse DHCP servers (interface with static IP, wan is excluded) + result = subprocess.check_output(['/usr/libexec/rpcd/ns.dhcp', 'call', 'list-interfaces']) + interfaces = json.loads(result) + for i in interfaces: + if uci.get('dhcp', i, 'ns_binding', default='') == '' or uci.get('dhcp', i, 'ns_binding', default='') == '0': + ret['disabled'] += 1 # Increment the count for DHCP servers + elif uci.get('dhcp', i, 'ns_binding', default='') == '1': + ret['soft-binding'] += 1 + elif uci.get('dhcp', i, 'ns_binding', default='') == '2': + ret['hard-binding'] += 1 + return ret + +def fact_backups(uci: EUci): + ret = { 'backup_passphrase': False, 'passphrase_date': 0 } + try: + ret['backup_passphrase'] = os.path.exists('/etc/backup.pass') + ret['passphrase_date'] = int(os.path.getmtime('/etc/backup.pass')) + except: + return {} + return ret + +def fact_ha(uci: EUci): + vrrp_instances = utils.get_all_by_type(uci, 'keepalived', 'vrrp_instance') + if vrrp_instances is None: + vrrp_instances = [] + ipaddresses = utils.get_all_by_type(uci, 'keepalived', 'ipaddress') + if ipaddresses is None: + ipaddresses = [] + + return { 'enabled': len(vrrp_instances) > 0, 'vips': len(ipaddresses) } + +def fact_default_password(uci: EUci): + data = { + 'username': 'root', + 'password': 'Nethesis,1234', + 'timeout': 1 + } + result = subprocess.run(['/bin/ubus', 'call', 'session', 'login', json.dumps(data)], capture_output=True) + + return { 'default_password': result.returncode == 0 } + +## Info +## These functions are used to send info about system to new my.nethesis.it and phonehome.nethserver.org + +def info_fqdn(uci: EUci): + system = uci.get_all('system') + for section in system: + for option in system[section]: + if option == 'hostname': + return anonymize(system[section][option], uci) + return '' + +def info_timezone(uci: EUci): + system = uci.get_all('system') + for section in system: + for option in system[section]: + if option == 'zonename': + return system[section][option] + return '' + +def info_kernel_version(uci: EUci): + try: + with open('/proc/version', 'r') as f: + version = f.read().strip() + return version.split()[2] + except: + return '' + +def info_uptime_seconds(uci: EUci): + try: + with open('/proc/uptime', 'r') as f: + uptime = f.read().strip().split()[0] + return floor(float(uptime)) + except: + return 0 + +def info_default_ipv4(uci: EUci): + # first method: dig -4 TXT +short o-o.myaddr.l.google.com @ns1.google.com + try: + res = subprocess.run(['dig', '-4', '+short', 'myip.opendns.com', '@resolver1.opendns.com'], + capture_output=True, text=True, timeout=3) + if res.returncode == 0 and res.stdout.strip(): + ip = res.stdout.strip().strip('"') + if ip: + return anonymize(ip, uci) + except: + pass + + # second method: curl -4 ifconfig.co + try: + res = subprocess.run(['curl', '-4', '-s', 'ifconfig.co'], + capture_output=True, text=True, timeout=3) + if res.returncode == 0 and res.stdout.strip(): + return anonymize(res.stdout.strip(), uci) + except: + pass + + # third method: get the first WAN device and its IPv4 + try: + wan_devices = utils.get_all_wan_devices(uci) + if wan_devices: + first_wan = wan_devices[0] + res = subprocess.run(['ip', '-4', '-j', 'addr', 'show', first_wan], + capture_output=True, text=True, timeout=3) + if res.returncode == 0: + addr_info = json.loads(res.stdout) + if addr_info and 'addr_info' in addr_info[0]: + for addr in addr_info[0]['addr_info']: + if addr.get('family') == 'inet': + return anonymize(addr.get('local'), uci) + except: + pass + + return '' + +def info_default_ipv6(uci: EUci): + # Get the first WAN device and its IPv6 + ipv6 = '' + try: + wan_devices = utils.get_all_wan_devices(uci) + if wan_devices: + first_wan = wan_devices[0] + res = subprocess.run(['ip', '-6', '-j', 'addr', 'show', first_wan], + capture_output=True, text=True, timeout=3) + if res.returncode == 0: + addr_info = json.loads(res.stdout) + if addr_info and 'addr_info' in addr_info[0]: + for addr in addr_info[0]['addr_info']: + if addr.get('family') == 'inet6' and addr.get('scope') == 'global': + ipv6 = addr.get('local') + except: + pass + + # If WAN has no IPv6, assume that IPv6 is not configured: speedup data collection + if ipv6 == '': + return '' + + # first method: dig -6 TXT +short o-o.myaddr.l.google.com @ns1.google.com + try: + res = subprocess.run(['dig', '-6', 'TXT', '+short', 'o-o.myaddr.l.google.com', '@ns1.google.com'], + capture_output=True, text=True, timeout=3) + if res.returncode == 0 and res.stdout.strip(): + ip = res.stdout.strip().strip('"') + if ip: + return anonymize(ip, uci) + except: + pass + + # second method: curl -6 ifconfig.co + try: + res = subprocess.run(['curl', '-6', '-s', 'ifconfig.co'], + capture_output=True, text=True, timeout=3) + if res.returncode == 0 and res.stdout.strip(): + return anonymize(res.stdout.strip(), uci) + except: + pass + + + return anonymize(ipv6, uci) + +def info_package_updates_available(uci: EUci): + """Check if package updates are available""" + try: + res = subprocess.run(['/usr/libexec/rpcd/ns.update', 'call', 'check-package-updates'], + capture_output=True, text=True, timeout=10) + if res.returncode == 0: + data = json.loads(res.stdout) + if isinstance(data, dict) and 'updates' in data: + updates = data['updates'] + if isinstance(updates, list) and len(updates) > 0: + return True + except: + pass + return False + +def info_image_updates_available(uci: EUci): + """Check if system image updates are available""" + try: + res = subprocess.run(['/usr/libexec/rpcd/ns.update', 'call', 'check-system-update'], + capture_output=True, text=True, timeout=10) + if res.returncode == 0: + data = json.loads(res.stdout) + current_version = data.get('currentVersion', '') + last_version = data.get('lastVersion', '') + + if current_version and last_version: + current = parse_version(current_version) + last = parse_version(last_version) + if current and last and last > current: + return True + except: + pass + return False + +def info_dns_servers(uci: EUci): + dns_list = [] + try: + for server in uci.get('dhcp', 'ns_dnsmasq', 'server', list=True, default=[]): + dns_list.append(server) + except: + pass + if not dns_list: + try: + with open('/tmp/resolv.conf.d/resolv.conf.auto', 'r') as f: + for line in f: + line = line.strip() + if line.startswith('nameserver'): + parts = line.split() + if len(parts) > 1: + dns_list.append(parts[1]) + except: + pass + return dns_list diff --git a/packages/python3-nethsec/src/nethsec/ipsec/__init__.py b/packages/python3-nethsec/src/nethsec/ipsec/__init__.py new file mode 100644 index 000000000..254966e6c --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/ipsec/__init__.py @@ -0,0 +1,93 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +IPSec utilities +''' + +import os +from nethsec import utils, firewall + +IPSEC_ZONE='ipsec' + +def init_ipsec(uci): + ''' + Initialize IPSec global configuration, if needed. + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + ''' + # Make sure the config file exists + conf = os.path.join(uci.confdir(), 'ipsec') + if not os.path.isfile(conf): + with open(conf, 'a'): + pass + + # Setup global options + gsettings = utils.get_id("ipsec_global") + uci.set("ipsec", gsettings, IPSEC_ZONE) + uci.set("ipsec", gsettings, "debug", '0') + uci.set("ipsec", gsettings, "zone", 'ipsec') + uci.commit('ipsec') + +def open_firewall_ports(uci): + ''' + Open firewall ports for IPSec tunnels, if need. + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + ''' + esp_accepted = False + ike_accepted = False + nat_accepted = False + esp = {"src": "wan", "dest_port": "", "proto": "esp", "target": "ACCEPT"} + ike = {"src": "wan", "dest_port": "500", "proto": "udp", "target": "ACCEPT"} + nat = {"src": "wan", "dest_port": "4500", "proto": "udp", "target": "ACCEPT"} + # search for existing rules + for r in utils.get_all_by_type(uci, 'firewall', 'rule'): + tmp = dict() + for opt in ['src', 'dest', 'dest_port', 'proto', 'target']: + tmp[opt] = uci.get('firewall', r, opt, default='') + # check if tmp is the esp rule + if all((tmp.get(k) == v for k, v in esp.items())): + esp_accepted = True + # check if tmp is the ike rule + if all((tmp.get(k) == v for k, v in ike.items())): + ike_accepted = True + # check if tmp is the nat rule + if all((tmp.get(k) == v for k, v in nat.items())): + nat_accepted = True + + if not ike_accepted: + firewall.add_template_rule(uci, 'ns_ipsec_ike') + + if not esp_accepted: + firewall.add_template_rule(uci, 'ns_ipsec_esp') + + if not nat_accepted: + firewall.add_template_rule(uci, 'ns_ipsec_nat') + + if not nat_accepted or not ike_accepted or not esp_accepted: + uci.save('firewall') + +def add_trusted_interface(uci, interface): + ''' + Add the interface to the 'ipsec' trusted zone. The function also creates the trusted zone, if needed. + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + ''' + if firewall.zone_exists(uci, IPSEC_ZONE): + firewall.add_interface_to_zone(uci, interface, IPSEC_ZONE) + else: + firewall.add_trusted_zone(uci, IPSEC_ZONE, [interface]) diff --git a/packages/python3-nethsec/src/nethsec/ldif/__init__.py b/packages/python3-nethsec/src/nethsec/ldif/__init__.py new file mode 100644 index 000000000..0c5e4c6b1 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/ldif/__init__.py @@ -0,0 +1,418 @@ +# Imported from: https://github.com/abilian/ldif + +# Copyright (c) 2019-2021, Abilian SAS +# Copyright (c) 2015, Tobias Bengfort +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""ldif - generate and parse LDIF data (see RFC 2849).""" + + +import base64 +import logging +import re +from collections import OrderedDict +from collections.abc import Collection, Iterator +from typing import Optional, Union +from urllib.parse import urlparse +from urllib.request import urlopen + +__version__ = "4.1.0" + +__all__ = [ + # constants + "LDIF_PATTERN", + # classes + "LDIFWriter", + "LDIFParser", +] + +log = logging.getLogger("ldif") + +ATTRTYPE_PATTERN = r"[\w;.-]+(;[\w_-]+)*" +ATTRVALUE_PATTERN = r'(([^,]|\\,)+|".*?")' +ATTR_PATTERN = ATTRTYPE_PATTERN + r"[ ]*=[ ]*" + ATTRVALUE_PATTERN +RDN_PATTERN = ATTR_PATTERN + r"([ ]*\+[ ]*" + ATTR_PATTERN + r")*[ ]*" +DN_PATTERN = RDN_PATTERN + r"([ ]*,[ ]*" + RDN_PATTERN + r")*[ ]*" +DN_REGEX = re.compile(f"^{DN_PATTERN}$") + +LDIF_PATTERN = ( + f"^((dn(:|::) {vars()['DN_PATTERN']})|(%(ATTRTYPE_PATTERN) s(:|::) .*)$)+" +) + +MOD_OPS = ["add", "delete", "replace"] +CHANGE_TYPES = ["add", "delete", "modify", "modrdn"] + + +def is_dn(s: str) -> bool: + """Return True if s is a LDAP DN.""" + if s == "": + return True + rm = DN_REGEX.match(s) + return rm is not None and rm.group(0) == s + + +UNSAFE_STRING_PATTERN = ( + "(^[^\x01-\x09\x0b-\x0c\x0e-\x1f\x21-\x39\x3b\x3d-\x7f]" + "|[^\x01-\x09\x0b-\x0c\x0e-\x7f])" +) +UNSAFE_STRING_RE = re.compile(UNSAFE_STRING_PATTERN) + + +def lower(string_list: Optional[Collection[str]]) -> list[str]: + """Return a list with the lowercased items of l.""" + return [i.lower() for i in string_list or []] + + +class LDIFWriter: + """Write LDIF entry or change records to file object. + + :type output_file: file-like object in binary mode + :param output_file: File for output + + :type base64_attrs: List[string] + :param base64_attrs: List of attribute types to be base64-encoded in any + case + + :type cols: int + :param cols: Specifies how many columns a line may have before it is + folded into many lines + + :type line_sep: bytearray + :param line_sep: line separator + + :type encoding: string + :param encoding: Encoding to use for converting values to bytes. Note that + the spec requires the dn field to be UTF-8 encoded, so it does not + really make sense to use anything else. Default: ``'utf8'``. + """ + + def __init__( + self, output_file, base64_attrs=(), cols=76, line_sep=b"\n", encoding="utf8" + ): + self._output_file = output_file + self._base64_attrs = lower(base64_attrs) + self._cols = cols + self._line_sep = line_sep + self._encoding = encoding + + self.records_written = 0 #: number of records that have been written + + def _fold_line(self, line: bytes) -> None: + """Output string line as one or more folded lines.""" + if len(line) <= self._cols: + self._output_file.write(line) + self._output_file.write(self._line_sep) + else: + pos = self._cols + self._output_file.write(line[0 : self._cols]) + self._output_file.write(self._line_sep) + while pos < len(line): + self._output_file.write(b" ") + end = min(len(line), pos + self._cols - 1) + self._output_file.write(line[pos:end]) + self._output_file.write(self._line_sep) + pos = end + + def _needs_base64_encoding(self, attr_type, attr_value): + """Return True if attr_value has to be base-64 encoded. + + This is the case because of special chars or because attr_type + is in self._base64_attrs + """ + return ( + attr_type.lower() in self._base64_attrs + or isinstance(attr_value, bytes) + or UNSAFE_STRING_RE.search(attr_value) is not None + ) + + def _unparse_attr(self, attr_type, attr_value): + """Write a single attribute type/value pair.""" + if self._needs_base64_encoding(attr_type, attr_value): + if not isinstance(attr_value, bytes): + attr_value = attr_value.encode(self._encoding) + encoded = base64.encodebytes(attr_value).replace(b"\n", b"").decode("ascii") + line = ":: ".join([attr_type, encoded]) + else: + line = ": ".join([attr_type, attr_value]) + self._fold_line(line.encode("ascii")) + + def _unparse_entry_record(self, entry): + """ + :type entry: Dict[string, List[string]] + :param entry: Dictionary holding an entry + """ + for attr_type in entry.keys(): + for attr_value in entry[attr_type]: + self._unparse_attr(attr_type, attr_value) + + def _unparse_changetype(self, mod_len): + """Detect and write the changetype.""" + if mod_len == 2: + changetype = "add" + elif mod_len == 3: + changetype = "modify" + else: + raise ValueError("modlist item of wrong length") + + self._unparse_attr("changetype", changetype) + + def _unparse_change_record(self, modlist): + """ + :type modlist: List[Tuple] + :param modlist: List of additions (2-tuple) or modifications (3-tuple) + """ + mod_len = len(modlist[0]) + self._unparse_changetype(mod_len) + + for mod in modlist: + if len(mod) != mod_len: + raise ValueError("Subsequent modlist item of wrong length") + + if mod_len == 2: + mod_type, mod_vals = mod + elif mod_len == 3: + mod_op, mod_type, mod_vals = mod + self._unparse_attr(MOD_OPS[mod_op], mod_type) + else: + raise ValueError(f"modlist item of wrong length: {mod_len}") + + for mod_val in mod_vals: + self._unparse_attr(mod_type, mod_val) + + if mod_len == 3: + self._output_file.write(b"-" + self._line_sep) + + def unparse(self, dn: str, record: Union[list, dict]): + """Write an entry or change record to the output file. + + :type dn: string + :param dn: distinguished name + :type record: Union[Dict[string, List[string]], List[Tuple]] + :param record: Either a dictionary holding an entry or a list of + additions (2-tuple) or modifications (3-tuple). + """ + self._unparse_attr("dn", dn) + if isinstance(record, dict): + self._unparse_entry_record(record) + elif isinstance(record, list): + self._unparse_change_record(record) + else: + raise TypeError("Argument record must be dictionary or list") + self._output_file.write(self._line_sep) + self.records_written += 1 + + +class LDIFParser: + """Read LDIF entry or change records from file object. + + :type input_file: file-like object in binary mode + :param input_file: file to read the LDIF input from + + :type ignored_attr_types: List[string] + :param ignored_attr_types: List of attribute types that will be ignored + + :type process_url_schemes: List[bytearray] + :param process_url_schemes: List of URL schemes to process with urllib. + An empty list turns off all URL processing and the attribute is + ignored completely. + + :type line_sep: bytearray + :param line_sep: line separator + + :type encoding: string + :param encoding: Encoding to use for converting values to unicode strings. + If decoding failes, the raw bytestring will be used instead. You can + also pass ``None`` which will skip decoding and always produce + bytestrings. Note that this only applies to entry values. ``dn`` and + entry keys will always be unicode strings. + + :type strict: boolean + :param strict: If set to ``False``, recoverable parse errors will produce + log warnings rather than exceptions. + """ + + def __init__( + self, + input_file, + ignored_attr_types=(), + process_url_schemes=(), + line_sep=b"\n", + encoding: Optional[str] = "utf8", + strict=True, + ): + self._input_file = input_file + self._process_url_schemes = lower(process_url_schemes) + self._ignored_attr_types = lower(ignored_attr_types) + self._line_sep = line_sep + self._encoding = encoding + self._strict = strict + + self.line_counter = 0 #: number of lines that have been read + self.byte_counter = 0 #: number of bytes that have been read + self.records_read = 0 #: number of records that have been read + + def parse(self) -> Iterator[tuple[Optional[str], dict]]: + """Iterate LDIF entry records. + + :rtype: Iterator[Tuple[string, Dict]] + :return: (dn, entry) + """ + for block in self._iter_blocks(): + yield self._parse_entry_record(block) + + @staticmethod + def _strip_line_sep(s): + """Strip trailing line separators from s, but no other whitespaces.""" + if s[-2:] == b"\r\n": + return s[:-2] + elif s[-1:] == b"\n": + return s[:-1] + else: + return s + + def _iter_unfolded_lines(self): + """Iter input unfoled lines. + + Skip comments. + """ + line = self._input_file.readline() + while line: + self.line_counter += 1 + self.byte_counter += len(line) + + line = self._strip_line_sep(line) + + nextline = self._input_file.readline() + while nextline and nextline[:1] == b" ": + line += self._strip_line_sep(nextline)[1:] + nextline = self._input_file.readline() + + if not line.startswith(b"#"): + yield line + line = nextline + + def _iter_blocks(self): + """Iter input lines in blocks separated by blank lines.""" + lines = [] + for line in self._iter_unfolded_lines(): + if line: + lines.append(line) + elif lines: + self.records_read += 1 + yield lines + lines = [] + if lines: + self.records_read += 1 + yield lines + + def _decode_value(self, attr_type, attr_value): + if attr_type == "dn": + try: + return attr_type, attr_value.decode("utf8") + except UnicodeError as err: + self._error(err) + return attr_type, attr_value.decode("utf8", "ignore") + + elif self._encoding is not None: + try: + return attr_type, attr_value.decode(self._encoding) + except UnicodeError: + pass + + return attr_type, attr_value + + def _parse_attr(self, line: bytes): + """Parse a single attribute type/value pair.""" + colon_pos = line.index(b":") + attr_type = line[0:colon_pos].decode("ascii") + + if line[colon_pos:].startswith(b"::"): + attr_value = base64.decodebytes(line[colon_pos + 2 :]) + elif line[colon_pos:].startswith(b":<"): + url = line[colon_pos + 2 :].strip() + attr_value = b"" + if self._process_url_schemes: + u = urlparse(url) + if u[0] in self._process_url_schemes: + attr_value = urlopen(url.decode("ascii")).read() + else: + attr_value = line[colon_pos + 1 :].strip() + + return self._decode_value(attr_type, attr_value) + + def _error(self, msg): + if self._strict: + raise ValueError(msg) + else: + log.warning(msg) + + def _check_dn(self, dn, attr_value): + """Check dn attribute for issues.""" + if dn is not None: + self._error("Two lines starting with dn: in one record.") + if not is_dn(attr_value): + self._error( + f"No valid string-representation of distinguished name {attr_value}." + ) + + def _check_changetype(self, dn, changetype, attr_value): + """Check changetype attribute for issues.""" + if dn is None: + self._error("Read changetype: before getting valid dn: line.") + if changetype is not None: + self._error("Two lines starting with changetype: in one record.") + if attr_value not in CHANGE_TYPES: + self._error(f"changetype value {attr_value} is invalid.") + + def _parse_entry_record(self, lines: list[bytes]) -> tuple[Optional[str], dict]: + """Parse a single entry record from a list of lines.""" + dn = None + entry: dict = OrderedDict() + + for line in lines: + attr_type, attr_value = self._parse_attr(line) + + if attr_type == "version" and dn is None: + continue # version = 1 + + if attr_type == "dn": + self._check_dn(dn, attr_value) + dn = attr_value + continue + + if dn is None: + self._error( + f"First line of record does not start with 'dn:': {attr_type}" + ) + continue + + if attr_type.lower() in self._ignored_attr_types: + continue + + if attr_type in entry: + entry[attr_type].append(attr_value) + else: + entry[attr_type] = [attr_value] + + return dn, entry diff --git a/packages/python3-nethsec/src/nethsec/mwan/__init__.py b/packages/python3-nethsec/src/nethsec/mwan/__init__.py new file mode 100644 index 000000000..b5585826f --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/mwan/__init__.py @@ -0,0 +1,698 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Mwan utilities +""" + + +import json +import subprocess + +import uci +from euci import EUci + +from nethsec import utils, objects +from nethsec.utils import ValidationError + +def _is_valid_src(e_uci: EUci, database_id: str): + """ + Validate the given object for source. + Source objects can be only: + - dhcp reservation + - dns domain + - vpn user + - a singleton host set + + Args: + e_uci: EUci instance + database_id: id of object + + Returns: + True if object is valid, False otherwise + """ + if objects.is_host_set(e_uci, database_id): + return objects.is_singleton_host_set(e_uci, database_id, allow_cidr=True) + + return objects.is_host(e_uci, database_id) or objects.is_domain(e_uci, database_id) or objects.is_vpn_user(e_uci, database_id) + +def _is_valid_dst(e_uci: EUci, database_id: str): + """ + Validate the given object for destination. + Destination objects can be only: + - dhcp reservation + - dns domain + - vpn user + - a singleton host set + + Args: + e_uci: EUci instance + database_id: id of object + + Returns: + True if object is valid, False otherwise + """ + if objects.is_host_set(e_uci, database_id): + return objects.is_singleton_host_set(e_uci, database_id, allow_cidr=True) + + return objects.is_host(e_uci, database_id) or objects.is_domain(e_uci, database_id) or objects.is_vpn_user(e_uci, database_id) + +def __generate_metric(e_uci: EUci) -> int: + """ + Generates a metric for an interface. + + Args: + e_uci: EUci instance + + Returns: + first metric that is not present in interface_metrics + """ + next_metric = 20 + for interface in utils.get_all_by_type(e_uci, 'network', 'interface').values(): + if 'metric' in interface: + next_metric = max(next_metric, int(interface['metric'])) + return next_metric + 10 + + +def __set_interface_defaults(e_uci: EUci, name: str): + default_interface_config = e_uci.get_all('ns-api', 'defaults_mwan') + e_uci.set('mwan3', name, 'initial_state', default_interface_config['initial_state']) + e_uci.set('mwan3', name, 'family', default_interface_config['protocol']) + e_uci.set('mwan3', name, 'track_ip', default_interface_config['track_ip']) + e_uci.set('mwan3', name, 'track_method', default_interface_config['tracking_method']) + e_uci.set('mwan3', name, 'reliability', default_interface_config['tracking_reliability']) + e_uci.set('mwan3', name, 'count', default_interface_config['ping_count']) + e_uci.set('mwan3', name, 'size', default_interface_config['ping_size']) + e_uci.set('mwan3', name, 'max_ttl', default_interface_config['ping_max_ttl']) + e_uci.set('mwan3', name, 'timeout', default_interface_config['ping_timeout']) + e_uci.set('mwan3', name, 'interval', default_interface_config['ping_interval']) + e_uci.set('mwan3', name, 'failure_interval', default_interface_config['ping_failure_interval']) + e_uci.set('mwan3', name, 'recovery_interval', default_interface_config['ping_recovery_interval']) + e_uci.set('mwan3', name, 'down', default_interface_config['interface_down_threshold']) + e_uci.set('mwan3', name, 'up', default_interface_config['interface_up_threshold']) + + +def __store_interface(e_uci: EUci, name: str) -> tuple[bool, bool]: + """ + Stores interface configuration for mwan3 and network, not suited to be used outside store_policy. + + Args: + e_uci: EUci instance + name: name of interface + + Returns: + tuple of booleans, first one indicates if mwan interface was created, second one indicates if metric was added + to network interface + + Raises: + ValidationError: if interface name is not defined in /etc/config/network + """ + # checking if interface is configured + try: + e_uci.get('network', name) + except uci.UciExceptionNotFound: + raise ValidationError('name', 'invalid', name) + + created_interface = False + # if no interface with name exists, create one with defaults + if e_uci.get('mwan3', name, default=None) is None: + created_interface = True + # fetch default configuration and set interface + e_uci.set('mwan3', name, 'interface') + e_uci.set('mwan3', name, 'enabled', '1') + __set_interface_defaults(e_uci, name) + + added_metric = False + # avoid adding metric if already present + if e_uci.get('network', name, 'metric', default=None) is None: + added_metric = True + # generate metric + metric = __generate_metric(e_uci) + # configure metric for interface + e_uci.set('network', name, 'metric', metric) + return created_interface, added_metric + + +def __store_member(e_uci: EUci, interface_name: str, metric: int, weight: int) -> tuple[str, bool]: + """ + Stores member configuration for mwan3, not suited to be used outside store_policy. + + Args: + e_uci: EUci instance + interface_name: name of interface to link the member to + metric: metric of the member + weight: weight of the member + + Returns: + tuple of string and boolean, first one is the generated name of the member, second one indicates if the member + was created + """ + member_config_name = utils.get_id(f'{interface_name}_M{metric}_W{weight}') + changed = False + if e_uci.get('mwan3', member_config_name, default=None) is None: + changed = True + e_uci.set('mwan3', member_config_name, 'member') + e_uci.set('mwan3', member_config_name, 'interface', interface_name) + e_uci.set('mwan3', member_config_name, 'metric', metric) + e_uci.set('mwan3', member_config_name, 'weight', weight) + return member_config_name, changed + + +def store_rule(e_uci: EUci, name: str, policy: str, protocol: str = None, + source_address: str = None, source_port: str = None, + destination_address: str = None, destination_port: str = None, sticky: bool = False, + ns_src: str = None, ns_dst: str = None) -> str: + """ + Stores a rule for mwan3 + + Args: + e_uci: EUci instance + name: name of the rule, must be unique + policy: policy to use for the rule, must be already set + protocol: must be one of tcp, udp, icmp or all, defaults to 'all' + source_address: source addresses to match + source_port: source ports to match or range + destination_address: destination addresses to match + destination_port: destination ports to match or range + sticky: whether to use sticky connections + ns_src: source object, it overrides source_address + ns_dst: destination object, it overrides destination_address + + Returns: + name of the rule created + + Raises: + ValidationError if name is not unique, policy is not valid or length get_id > 15 + """ + # check if the rule length is not more than 15 characters: 12 from user and 3 from get_id prefix + if len(name) > 12: + # get_id add 3 more chars (ns_) to the name + raise ValidationError('name', 'length_12_max', name) + rule_config_name = utils.get_id(name.lower(), 15) + rules = utils.get_all_by_type(e_uci, 'mwan3', 'rule').keys() + if e_uci.get('mwan3', rule_config_name, default=None) is not None: + raise ValidationError('name', 'unique', name) + if e_uci.get('mwan3', policy, default=None) is None: + raise ValidationError('policy', 'invalid', policy) + if ns_src and not _is_valid_src(e_uci, ns_src): + raise ValidationError('ns_src', 'invalid_object', ns_src) + if ns_dst and not _is_valid_dst(e_uci, ns_dst): + raise ValidationError('ns_dst', 'invalid_object', ns_dst) + + e_uci.set('mwan3', rule_config_name, 'rule') + e_uci.set('mwan3', rule_config_name, 'label', name) + e_uci.set('mwan3', rule_config_name, 'use_policy', policy) + # test if sticky is True or False, if not raise an error + if type(sticky) is not bool: + raise ValidationError('sticky', 'sticky_not_valid', sticky) + e_uci.set('mwan3', rule_config_name, 'sticky', sticky) + if protocol is not None: + e_uci.set('mwan3', rule_config_name, 'proto', protocol) + if source_address is not None: + e_uci.set('mwan3', rule_config_name, 'src_ip', source_address) + if source_port is not None: + e_uci.set('mwan3', rule_config_name, 'src_port', source_port.replace('-', ':')) + if destination_address is not None: + e_uci.set('mwan3', rule_config_name, 'dest_ip', destination_address) + if destination_port is not None: + e_uci.set('mwan3', rule_config_name, 'dest_port', destination_port.replace('-', ':')) + if ns_src is not None: + e_uci.set('mwan3', rule_config_name, 'ns_src', ns_src) + if ns_dst is not None: + e_uci.set('mwan3', rule_config_name, 'ns_dst', ns_dst) + + order_rules(e_uci, [rule_config_name] + list(rules)) + update_rules(e_uci) # update rules with objects and save mwan3 config + return f'mwan3.{rule_config_name}' + + +def store_policy(e_uci: EUci, name: str, interfaces: list[dict]) -> list[str]: + """ + Stores a policy for mwan3, takes care of creating interfaces and members. + + Args: + e_uci: EUci instance + name: name of policy + interfaces: list of interfaces to add to policy, must have name, metric and weight fields + + Returns: + list of changed configuration + + Raises: + ValidationError: if name is not unique or length or get_id > 15 + """ + changed_config = [] + if len(name) > 12: + # get_id add 3 more chars (ns_) to the name + raise ValidationError('name', 'length_12_max', name) + # generate policy name + policy_config_name = utils.get_id(name.lower(), 15) + # test length of policy name + # make sure name is not something that already exists + if e_uci.get('mwan3', policy_config_name, default=None) is not None: + raise ValidationError('name', 'unique', name) + # generate policy config with corresponding name + e_uci.set('mwan3', policy_config_name, 'policy') + e_uci.set('mwan3', policy_config_name, 'label', name) + changed_config.append(f'mwan3.{policy_config_name}') + + member_names = __add_interfaces(e_uci, interfaces, changed_config) + + e_uci.set('mwan3', policy_config_name, 'use_member', member_names) + + if len(utils.get_all_by_type(e_uci, 'mwan3', 'rule')) == 0: + changed_config.append(store_rule(e_uci, 'Default Rule', policy_config_name)) + + # default policy must have a last_resort field + if len(utils.get_all_by_type(e_uci, 'mwan3', 'policy')) == 1: + e_uci.set('mwan3', policy_config_name, 'last_resort', 'default') + + e_uci.save('mwan3') + e_uci.save('network') + return changed_config + + +def __fetch_interface_status(interface_name: str) -> str: + try: + output = (subprocess.run([ + 'ubus', + 'call', + 'mwan3', + 'status', + '{"section": "interfaces"}' + ], capture_output=True, check=True) + .stdout.decode('utf-8')) + decoded_output = json.JSONDecoder().decode(output) + return decoded_output['interfaces'][interface_name]['status'] + except: + return 'unknown' + + +def __parse_member(e_uci: EUci, member_name: str) -> dict: + """ + Parses a member configuration and returns formatted data. + + Args: + e_uci: EUci instance + member_name: member name + + Returns: + dict with member data + """ + member_data = e_uci.get_all('mwan3', member_name) + return { + 'name': member_name, + 'interface': member_data['interface'], + 'metric': member_data['metric'], + 'weight': member_data['weight'], + 'status': __fetch_interface_status(member_data['interface']) + } + + +def index_policies(e_uci: EUci) -> list[dict]: + """ + Returns a list of policies with their members, interfaces and metrics/weights. + + Args: + e_uci: EUci instance + + Returns: + list of dicts with policy data + """ + data = [] + policies = utils.get_all_by_type(e_uci, 'mwan3', 'policy') + # iterate over policies + for policy_name in policies.keys(): + policy = policies[policy_name] + policy_data = { + 'name': policy_name, + } + # add label only if present + if 'label' in policy: + policy_data['label'] = policy['label'] + + # add members + members = [] + if 'use_member' in policy: + policy_data['members'] = {} + for member in policy['use_member']: + members.append(__parse_member(e_uci, member)) + + # infer policy type by metrics + metrics = [int(member['metric']) for member in members] + if len(metrics) == 1: + policy_data['type'] = 'custom' + elif all(metric == metrics[0] for metric in metrics): + policy_data['type'] = 'balance' + elif all(metrics.index(metric) == key for key, metric in enumerate(metrics)): + policy_data['type'] = 'backup' + else: + policy_data['type'] = 'custom' + + unique_metrics = list(set(metrics)) + for unique_metric in unique_metrics: + policy_data['members'][unique_metric] = list(filter(lambda x: x['metric'] == str(unique_metric), members)) + + # append policy to data + data.append(policy_data) + return data + + +def __add_interfaces(e_uci: EUci, interfaces: list[dict], changed_config: list[str] = None) -> list[str]: + """ + Add interfaces to policy, takes care of creating interfaces and members. + + Args: + e_uci: euci instance + interfaces: list of interfaces to add + changed_config: array of changed configuration + + Returns: + list of member names added to policy + + Raises: + ValidationError: if interface name is not defined in /etc/config/network + """ + if changed_config is None: + changed_config = list() + member_names: list[str] = [] + for interface in interfaces: + try: + added_mwan_interface, updated_interface = __store_interface(e_uci, interface['name']) + except ValidationError: + raise ValidationError('interfaces', 'invalid', interface['name']) + if added_mwan_interface: + changed_config.append(f'mwan3.{interface["name"]}') + if updated_interface: + changed_config.append(f'network.{interface["name"]}') + + member_config_name, member_created = __store_member(e_uci, + interface['name'], + interface['metric'], + interface['weight']) + member_names.append(member_config_name) + if member_created: + changed_config.append(f'mwan3.{member_config_name}') + + return member_names + + +def edit_policy(e_uci: EUci, name: str, label: str, interfaces: list[dict]) -> list[str]: + """ + Edits a mwan3 policy. + + Args: + e_uci: euci instance + name: name of policy to edit + label: policy label + interfaces: dict of interfaces to add to policy + + Returns: + list of changed mwan3 entries + + Raises: + ValidationError: if name is not valid + """ + if e_uci.get('mwan3', name, default=None) is None: + raise ValidationError('name', 'invalid', name) + changed_config = [] + if label != e_uci.get_all('mwan3', name)['label']: + e_uci.set('mwan3', name, 'label', label) + changed_config.append(f'mwan3.{name}') + + member_names = __add_interfaces(e_uci, interfaces, changed_config) + + e_uci.set('mwan3', name, 'use_member', member_names) + + e_uci.save('mwan3') + e_uci.save('network') + return changed_config + + +def delete_policy(e_uci: EUci, name: str) -> list[str]: + """ + Deletes a mwan3 policy. + + Args: + e_uci: euci instance + name: name of policy to delete + + Returns: + list of deleted mwan3 entries + """ + if e_uci.get('mwan3', name, default=None) is None: + raise ValidationError('name', 'invalid', name) + e_uci.delete('mwan3', name) + e_uci.save('mwan3') + return [f'mwan3.{name}'] + + +def index_rules(e_uci: EUci) -> list[dict]: + """ + Returns a list of rules with their policies. + + Args: + e_uci: euci instance + + Returns: + parsed list of rules with assigned policy + """ + data = [] + rules = utils.get_all_by_type(e_uci, 'mwan3', 'rule') + for rule_key in rules.keys(): + rule_data = {} + rule_value = rules[rule_key] + rule_data['name'] = rule_key + rule_data['policy'] = {} + rule_data['policy']['name'] = rule_value['use_policy'] + if e_uci.get('mwan3', rule_value['use_policy'], default=None) is not None: + rule_data['policy']['label'] = utils.get_all_by_type(e_uci, 'mwan3', 'policy')[rule_value['use_policy']].get('label') + if 'label' in rule_value: + rule_data['label'] = rule_value['label'] + if 'proto' in rule_value: + rule_data['protocol'] = rule_value['proto'] + if 'src_ip' in rule_value: + rule_data['source_address'] = rule_value['src_ip'] + if 'src_port' in rule_value: + rule_data['source_port'] = rule_value['src_port'].replace(':', '-') + if 'dest_ip' in rule_value: + rule_data['destination_address'] = rule_value['dest_ip'] + if 'dest_port' in rule_value: + rule_data['destination_port'] = rule_value['dest_port'].replace(':', '-') + if 'sticky' in rule_value: + rule_data['sticky'] = rule_value['sticky'] == '1' + if 'ns_src' in rule_value: + rule_data['ns_src'] = rule_value['ns_src'] + if 'ns_dst' in rule_value: + rule_data['ns_dst'] = rule_value['ns_dst'] + + data.append(rule_data) + return data + + +def order_rules(e_uci: EUci, rules: list[str]) -> list[str]: + """ + Orders mwan3 rules, moves everything else but rules to the end of the list. + + Args: + e_uci: euci instance + rules: which order to put rules + + Returns: + list of ordered mwan3 entries + + Raises: + ValidationError: if a rule is not present in /etc/config/mwan3 + """ + for rule in utils.get_all_by_type(e_uci, 'mwan3', 'rule').keys(): + if rule not in rules: + raise ValidationError('rules', 'missing', rule) + + order: list[str] = [] + + for key in e_uci.get_all('mwan3').keys(): + if key not in rules: + order.append(key) + + order.extend(rules) + + subprocess.run([ + 'ubus', + 'call', + 'uci', + 'order', + json.dumps({ + 'config': 'mwan3', + 'sections': order, + }) + ]) + + e_uci.save('mwan3') + return order + + +def delete_rule(e_uci: EUci, name: str): + """ + Deletes a mwan3 rule. + + Args: + e_uci: euci instance + name: rule name to delete + + Returns: + name of rule deleted + """ + if e_uci.get('mwan3', name, default=None) is None: + raise ValidationError('name', 'invalid', name) + + e_uci.delete('mwan3', name) + e_uci.save('mwan3') + return f'mwan3.{name}' + + +def edit_rule(e_uci: EUci, name: str, policy: str, label: str, protocol: str = None, + source_address: str = None, source_port: str = None, + destination_address: str = None, destination_port: str = None, sticky: bool = False, + ns_src: str = None, ns_dst: str = None): + """ + Edits a mwan3 rule. + + Args: + e_uci: EUci instance + name: rule name + policy: policy to use for the rule + label: rule label + protocol: protocol in which the rule applies + source_address: CIDR notation of source address + source_port: port or port range + destination_address: CIDR notation of destination address + destination_port: port or port range + sticky: whether to use sticky connections + ns_src: source object, it overrides source_address + ns_dst: destination object, it overrides destination_address + + Raises: + ValidationError: if name is not valid or policy is not valid + """ + if e_uci.get('mwan3', name, default=None) is None: + raise ValidationError('name', 'invalid', name) + + if e_uci.get('mwan3', policy, default=None) is None: + raise ValidationError('policy', 'invalid', policy) + if ns_src and not _is_valid_src(e_uci, ns_src): + raise ValidationError('ns_src', 'invalid_object', ns_src) + if ns_dst and not _is_valid_dst(e_uci, ns_dst): + raise ValidationError('ns_dst', 'invalid_object', ns_dst) + e_uci.set('mwan3', name, 'use_policy', policy) + e_uci.set('mwan3', name, 'label', label) + # test if sticky is True of False, if not raise an error + if type(sticky) is not bool: + raise ValidationError('sticky', 'sticky_not_valid', sticky) + e_uci.set('mwan3', name, 'sticky', sticky) + if protocol is not None: + e_uci.set('mwan3', name, 'proto', protocol) + if protocol != 'tcp' and protocol != 'udp': + e_uci.delete('mwan3', name, 'src_port') + e_uci.delete('mwan3', name, 'dest_port') + else: + if destination_port is not None: + e_uci.set('mwan3', name, 'dest_port', destination_port.replace('-', ':')) + if source_port is not None: + e_uci.set('mwan3', name, 'src_port', source_port.replace('-', ':')) + if source_address is not None: + e_uci.set('mwan3', name, 'src_ip', source_address) + else: + e_uci.delete('mwan3', name, 'src_ip') + if destination_address is not None: + e_uci.set('mwan3', name, 'dest_ip', destination_address) + else: + e_uci.delete('mwan3', name, 'dest_ip') + if ns_src is not None: + e_uci.set('mwan3', name, 'ns_src', ns_src) + else: + e_uci.delete('mwan3', name, 'ns_src') + if ns_dst is not None: + e_uci.set('mwan3', name, 'ns_dst', ns_dst) + else: + e_uci.delete('mwan3', name, 'ns_dst') + update_rules(e_uci) # update rules with objects and save mwan3 config + return f'mwan3.{name}' + + +def clear_config(e_uci: EUci): + """ + Clears mwan3 configuration. + + Args: + e_uci: euci instance + """ + for entry in e_uci.get_all('mwan3'): + if entry != 'globals': + e_uci.delete('mwan3', entry) + + e_uci.save('mwan3') + + +def set_default_config(e_uci: EUci, track_ip: list[str], ping_timeout: int, ping_interval: int, + ping_failure_interval: int, interface_down_threshold: int, interface_up_threshold: int): + """ + Sets default configuration for mwan3. + + Args: + e_uci: euci instance + track_ip: list of IPs to track + ping_timeout: timeout for ping + ping_interval: interval between pings + ping_failure_interval: interval between failed pings + interface_down_threshold: threshold for interface down + interface_up_threshold: threshold for interface up + """ + e_uci.set('ns-api', 'defaults_mwan', 'track_ip', track_ip) + e_uci.set('ns-api', 'defaults_mwan', 'ping_timeout', ping_timeout) + e_uci.set('ns-api', 'defaults_mwan', 'ping_interval', ping_interval) + e_uci.set('ns-api', 'defaults_mwan', 'ping_failure_interval', ping_failure_interval) + e_uci.set('ns-api', 'defaults_mwan', 'interface_down_threshold', interface_down_threshold) + e_uci.set('ns-api', 'defaults_mwan', 'interface_up_threshold', interface_up_threshold) + + for interface in utils.get_all_by_type(e_uci, 'mwan3', 'interface'): + __set_interface_defaults(e_uci, interface) + + e_uci.save('ns-api') + e_uci.save('mwan3') + + +def get_default_config(e_uci: EUci) -> dict: + """ + Returns default configuration for mwan3. + + Args: + e_uci: euci instance + + Returns: + dict with default configuration + """ + return e_uci.get_all('ns-api', 'defaults_mwan') + + +def update_rules(e_uci: EUci): + """ + Updates mwan3 rules with objects addresses + + Args: + e_uci: euci instance + """ + for rule in utils.get_all_by_type(e_uci, 'mwan3', 'rule'): + ns_src = e_uci.get('mwan3', rule, 'ns_src', default=None) + ns_dst = e_uci.get('mwan3', rule, 'ns_dst', default=None) + # both ns_src and ns_dst should be a singleton + if ns_src: + e_uci.set('mwan3', rule, 'src_ip', objects.get_object_ip(e_uci, ns_src)) + if ns_dst: + # domain sets are not supported because mwan3 ipset functionality is broken on 23.05. + e_uci.set('mwan3', rule, 'dest_ip', objects.get_object_ip(e_uci, ns_dst)) + + e_uci.save('mwan3') diff --git a/packages/python3-nethsec/src/nethsec/objects/__init__.py b/packages/python3-nethsec/src/nethsec/objects/__init__.py new file mode 100644 index 000000000..ece401749 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/objects/__init__.py @@ -0,0 +1,884 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +Objects utilities +''' +import ipaddress +import json +import os +import subprocess + +from euci import EUci + +from nethsec import utils, firewall + +# Generic + +def is_object_id(id): + """ + Check if an id is an object id. + + Args: + id: id to check + + Returns: + True if id is an object id, False otherwise + """ + if not id or not isinstance(id, str): + return False + return id.startswith('objects/') or id.startswith('dhcp/') or id.startswith('users/') + +def object_exists(uci, database_id): + """ + Check if the object exists. + + Args: + database_id: id to check in the form of `/` + + Returns: + True if object exists, False otherwise + """ + try: + database, id = database_id.split('/') + uci.get(database, id) + return True + except: + return False + +def get_object(uci, database_id): + """ + Get object from objects config. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + object from config or None if not found + """ + try: + database, id = database_id.split('/') + return uci.get_all(database, id) + except: + return None + +def is_used_object(uci, database_id): + """ + Check if an object is used in: + - firewall config + - mwan3 config + - dpi config + - existing host set + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + A tuple with: + - True if domain set is used in firewall config, False otherwise + - a list of firewall sections where domain set is used + """ + matches = [] + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_src', default=None) == database_id or uci.get('firewall', section, 'ns_dst', default=None) == database_id: + matches.append(f'firewall/{section}') + for section in uci.get_all("mwan3"): + if uci.get('mwan3', section, 'ns_src', default=None) == database_id or uci.get('mwan3', section, 'ns_dst', default=None) == database_id: + matches.append(f'mwan3/{section}') + for section in uci.get_all("dpi"): + if uci.get('dpi', section, 'source', default=None) == database_id: + matches.append(f'dpi/{section}') + for section in uci.get_all("objects"): + try: + ips = uci.get_all('objects', section, 'ipaddr') + if database_id in ips: + matches.append(f'objects/{section}') + except: + continue + return len(matches) > 0, matches + +def get_object_ips(uci, database_id): + """ + Get all IP addresses from an object. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + a list of unique IP addresses from the object + """ + ips = [] + if not database_id: + return ips + + obj = get_object(uci, database_id) + database, id = database_id.split('/') + + if not obj: + return ips + + if database == 'dhcp': + ip = obj.get('ip') + if ip: + ips.append(ip) + elif database == 'users': + openvpn_ipaddr = obj.get('openvpn_ipaddr') + if openvpn_ipaddr: + ips.append(openvpn_ipaddr) + elif database == 'objects': + ipaddr = obj.get('ipaddr') + if ipaddr: + for ip in ipaddr: + if is_object_id(ip): + ips.extend(get_object_ips(uci, ip)) + else: + ips.append(ip) + + return list(set(ips)) # Convert the list to a set to remove duplicates, then convert it back to a list + +def get_object_ip(uci, database_id): + """ + Get the first IP address from an object. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + the first IP address from the object + """ + ips = get_object_ips(uci, database_id) + if ips: + return ips[0] + return None + +# Domain set + +def is_domain_set(uci, database_id): + """ + Check if an object is a domain set. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + True if object is a domain set, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database =="objects" and obj_type == "domain" + except: + return False + +def is_used_domain_set(uci, id): + """ + Check if domain set is used in firewall config. + + Args: + uci: EUci pointer + id: id of domain set + + Returns: + A tuple with: + - True if domain set is used in firewall config, False otherwise + - a list of firewall sections where domain set is used + """ + return is_used_object(uci, f'objects/{id}') + +def get_domain_set_ipsets(uci, id): + """ + Get ipsets linked to domain set. + + Args: + uci: EUci pointer + id: id of domain set + + Returns: + a dictionary with + - `firewall`: the ipset id linked to domain set from firewall config + - `dhcp`: the ipset id linked to domain set from dhcp config + """ + ipsets = {"firewall": None, "dhcp": None} + for section in utils.get_all_by_type(uci, "firewall", "ipset"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + ipsets["firewall"] = uci.get("firewall", section, "name", default='') + break + for section in utils.get_all_by_type(uci, "dhcp", "ipset"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + ipsets["dhcp"] = section + break + return ipsets + +def get_domain_set_total_chars(name: str, unique_domains: list[str]): + """ + Calculate the total number of characters the given domain set will use in the dnsmasq configuration file. + This file supports a maximum of 1024 characters per line. + + Example syntax of a domain set in dnsmasq configuration file: + nftset=/example1.org/example2.org/example3.org/.../4#inet#fw4#domainSetName + + Args: + name: name of domain set + domains: a list of valid DNS names + + Returns: + the number of characters + """ + total_chars = 8 + 11 # leading "nftset=/" and trailing "4#inet#fw4#" + + for domain in unique_domains: + total_chars += len(domain) + 1 # +1 for trailing "/" + + total_chars += len(name) # domain set name + return total_chars + +def add_domain_set(uci, name: str, family: str, domains: list[str], timeout: int = 660) -> str: + """ + Add domain set to objects config. + + Args: + uci: EUci pointer + name: name of domain set + family: can be `ipv4` or `ipv6` + domains: a list of valid DNS names + timeout: the timeout in seconds for the DNS resolution, default is `660` seconds + + Returns: + id of domain set config that was added + """ + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + # check name contains only number and letters + if not name.isalnum(): + raise utils.ValidationError('name', 'invalid_name', name) + if family not in ['ipv4', 'ipv6']: + raise utils.ValidationError('family', 'invalid_family', family) + if timeout < 0: + raise utils.ValidationError('timeout', 'invalid_timeout', timeout) + + # remove duplicated domains + unique_domains = list(dict.fromkeys(domains)) + if get_domain_set_total_chars(name, unique_domains) > 1024 - 10: # -10 chars for safety + raise utils.ValidationError('domain', 'total_chars_exceeded') + + id = utils.get_random_id() + uci.set('objects', id, 'domain') + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'timeout', timeout) + uci.set('objects', id, 'domain', unique_domains) + uci.save('objects') + + # create ipset inside dhcp config + ipset = utils.get_random_id() + uci.set('dhcp', ipset, 'ipset') + uci.set('dhcp', ipset, 'name', [name]) + uci.set('dhcp', ipset, 'domain', unique_domains) + uci.set('dhcp', ipset, 'table_family', 'inet') + uci.set('dhcp', ipset, 'ns_link', f'objects/{id}') + uci.save('dhcp') + + # create ipset inside firewall config + ipset = utils.get_random_id() + uci.set('firewall', ipset, 'ipset') + uci.set('firewall', ipset, 'name', name) + uci.set('firewall', ipset, 'family', family) + uci.set('firewall', ipset, 'timeout', timeout) + uci.set('firewall', ipset, 'counters', '1') + uci.set('firewall', ipset, 'match', 'ip') + uci.set('firewall', ipset, 'ns_link', f'objects/{id}') + uci.save('firewall') + return id + +def edit_domain_set(uci, id: str, name: str, family: str, domains: list[str], timeout: int = 660) -> str: + """ + Edit domain set in objects config. + + Args: + uci: EUci pointer + id: id of domain set to edit + name: name of domain set + family: can be `ipv4` or `ipv6` + domains: a list of valid DNS names + timeout: the timeout in seconds for the DNS resolution, default is `660` seconds + + Returns: + id of domain set config that was edited + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "domain_set_does_not_exists", id) + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + if family not in ['ipv4', 'ipv6']: + raise utils.ValidationError('family', 'invalid_family', family) + if timeout < 0: + raise utils.ValidationError('timeout', 'invalid_timeout', timeout) + + # remove duplicated domains + unique_domains = list(dict.fromkeys(domains)) + if get_domain_set_total_chars(name, unique_domains) > 1024 - 10: # -10 chars for safety + raise utils.ValidationError('domain', 'total_chars_exceeded') + + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'timeout', timeout) + uci.set('objects', id, 'domain', unique_domains) + uci.save('objects') + + # update ipset inside dhcp config + for section in uci.get_all("dhcp"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + uci.set('dhcp', section, 'name', [name]) + uci.set('dhcp', section, 'domain', unique_domains) + uci.set('dhcp', section, 'ns_tag', ['automated']) + uci.save('dhcp') + break + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + uci.set('firewall', section, 'name', name) + uci.set('firewall', section, 'family', family) + uci.set('firewall', section, 'timeout', timeout) + uci.set('firewall', section, 'ns_tag', ['automated']) + uci.save('firewall') + break + return id + +def delete_domain_set(uci, id: str) -> str: + """ + Delete domain set from objects config. + + Args: + uci: EUci pointer + id: id of domain set to delete + + Returns: + name of domain set config that was deleted + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "domain_set_does_not_exists", id) + used, matches = is_used_domain_set(uci, id) + if used: + raise utils.ValidationError("id", "domain_set_is_used", matches) + uci.delete('objects', id) + uci.save('objects') + for section in uci.get_all("dhcp"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + uci.delete('dhcp', section) + uci.save('dhcp') + break + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + uci.delete('firewall', section) + uci.save('firewall') + break + return id + +def list_domain_sets(uci, used_info = True) -> list: + """ + Get all domain sets from objects config + + Args: + uci: EUci pointer + used_info: include used and matches info + + Returns: + a list of all domain sets + """ + sets = [] + for section in uci.get_all("objects"): + if uci.get('objects', section) == 'domain': + rule = uci.get_all('objects', section) + rule['id'] = section + rule['type'] = 'domain_set' + rule['subtype'] = 'domain_set' + if used_info: + used, matches = is_used_domain_set(uci, section) + rule['used'] = used + rule['matches'] = matches + sets.append(rule) + return sets + +# Host set + +def _has_loop(uci, id, ipaddr, depth=0): + if depth > 2: + return True + if is_object_id(ipaddr): + if ipaddr == id: + return True + obj = get_object(uci, ipaddr) + if obj: + for ip in obj.get('ipaddr', []): + if _has_loop(uci, id, ip, depth + 1): + return True + return False + +def _validate_host_set_ipaddr(uci, id, ipaddr: str, family: str): + if is_object_id(ipaddr): + if not object_exists(uci, ipaddr): + raise utils.ValidationError('ipaddr', 'object_does_not_exists', ipaddr) + else: + if id and is_host_set(uci, id): + # check loop + if _has_loop(uci, id, ipaddr): + raise utils.ValidationError('ipaddr', 'loop_detected', ipaddr) + return # validation is ok + + if family == 'ipv4': + return _validate_host_set_ipaddr_v4(ipaddr) + elif family == 'ipv6': + return _validate_host_set_ipaddr_v6(ipaddr) + +def _validate_host_set_ipaddr_v4(ipaddr: str): + if '/' in ipaddr: + # validate CIDR + try: + ipaddress.IPv4Network(ipaddr) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + elif '-' in ipaddr: + start, end = ipaddr.split('-') + try: + ipaddress.IPv4Address(start) + ipaddress.IPv4Address(end) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + else: + # validate IPv4 + try: + ipaddress.IPv4Address(ipaddr) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + return True + +def _validate_host_set_ipaddr_v6(ipaddr: str): + if '/' in ipaddr: + # validate CIDR + try: + ipaddress.IPv6Network(ipaddr) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + elif '-' in ipaddr: + start, end = ipaddr.split('-') + try: + ipaddress.IPv6Address(start) + ipaddress.IPv6Address(end) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + else: + # validate IPv6 + try: + ipaddress.IPv6Address(ipaddr) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + return True + +def add_host_set(uci, name: str, family: str, ipaddrs: list[str]) -> str: + """ + Add host set to objects config. + + Args: + uci: EUci pointer + name: name of host set + family: can be `ipv4` or `ipv6` + ipaddrs: a list of IP addresses + + Returns: + id of host set config that was added + """ + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + # check name contains only number and letters + if not name.isalnum(): + raise utils.ValidationError('name', 'invalid_name', name) + for ipaddr in ipaddrs: + _validate_host_set_ipaddr(uci, '', ipaddr, family) + id = utils.get_random_id() + uci.set('objects', id, 'host') + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'ipaddr', ipaddrs) + uci.save('objects') + return id + +def edit_host_set(uci, id: str, name: str, family: str, ipaddrs: list[str]) -> str: + """ + Edit host set in objects config. + + Args: + uci: EUci pointer + id: id of host set to edit + name: name of host set + family: can be `ipv4` or `ipv6` + ipaddrs: a list of IP addresses + + Returns: + id of host set config that was edited + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "host_set_does_not_exists", id) + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + for ipaddr in ipaddrs: + _validate_host_set_ipaddr(uci, f'objects/{id}', ipaddr, family) + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'ipaddr', ipaddrs) + uci.save('objects') + return id + +def delete_host_set(uci, id: str) -> str: + """ + Delete host set from objects config. + + Args: + uci: EUci pointer + id: id of host set to delete + + Returns: + name of host set config that was deleted + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "host_set_does_not_exists", id) + used, matches = is_used_host_set(uci, id) + if used: + raise utils.ValidationError("id", "host_set_is_used", matches) + uci.delete('objects', id) + uci.save('objects') + return id + +def is_used_host_set(uci, id): + """ + Check if host set is used in firewall config. + + Args: + uci: EUci pointer + id: id of host set + + Returns: + A tuple with: + - True if host set is used in firewall config, False otherwise + - a list of firewall sections where host set is used + """ + return is_used_object(uci, f'objects/{id}') + +def list_host_sets(uci, used_info = True) -> list: + """ + Get all host sets from objects config + + Args: + uci: EUci pointer + used_info: include used and matches info + + Returns: + a list of all host sets + """ + sets = [] + for section in uci.get_all("objects"): + if uci.get('objects', section) == 'host': + rule = uci.get_all('objects', section) + rule['id'] = section + rule['singleton'] = is_singleton_host_set(uci, f'objects/{section}') + if rule['singleton']: + # set subtype to CIDR, range or host + ip = get_object_ip(uci, f'objects/{section}') + if '/' in ip: + rule['subtype'] = 'cidr' + elif '-' in ip: + rule['subtype'] = 'range' + else: + rule['subtype'] = 'host' + else: + rule['subtype'] = 'host_set' + if used_info: + used, matches = is_used_host_set(uci, section) + rule['used'] = used + rule['matches'] = matches + sets.append(rule) + return sets + +def is_host_set(uci, database_id): + """ + Check if an object is a host set. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + True if object is a host set, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "objects" and obj_type == "host" + except: + return False + +# in mathematical terms, a singleton set is a set with exactly one element +def is_singleton_host_set(uci, database_id, allow_cidr=False): + """ + Check if an object is a host set with a single IP address. + The IP must not be an IP range. + If `allow_cidr` is True, the IP can be in CIDR notation. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + allow_cidr: allow CIDR notation + + Returns: + True if object is a singleton host set, False otherwise + """ + if is_host_set(uci, database_id): + obj = get_object(uci, database_id) + if obj and len(obj.get('ipaddr')) == 1: + ip = obj.get('ipaddr')[0] + if '-' in ip: + return False + if allow_cidr: + return True + else: + return '/' not in ip + return False + +# Host + +def is_host(uci, database_id): + """ + Check if an object is a host. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a host, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "dhcp" and obj_type == "host" + except: + return False + +# Domain + +def is_domain(uci, database_id): + """ + Check if an object is a domain. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a domain, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "dhcp" and obj_type == "domain" + except: + return False + +# VPN user + +def is_vpn_user(uci, database_id): + """ + Check if an object is a VPN user. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a VPN user, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "users" and obj_type == "user" and uci.get(database, id, 'openvpn_ipaddr', default=None) != None + except: + return False + +# API suggestions functions + +# Each element of the list should contain the following fields: +# - `id`: the id of the object +# - `name`: the name of the object +# - `type`: the type of the object +# - `family`: the family of the object (optional) +# If expand flag is set to True, the list should contain all IP addresses of the object + +def list_vpn_users(uci, expand=False, used_info=False): + """ + Get all VPN users from users config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + used_info: include used and matches info + + Returns: + a list of all VPN users + """ + users = [] + for section in uci.get_all("users"): + user = {} + if uci.get('users', section) == 'user' and uci.get('users', section, 'openvpn_ipaddr', default=None) != None: + obj = uci.get_all('users', section) + user['id'] = f"users/{section}" + user['name'] = obj.get('name') + user['type'] = 'vpn_user' + user['subtype'] = 'vpn_user' + user['family'] = 'ipv4' + if expand: + user['ipaddr'] = [obj.get('openvpn_ipaddr')] + if used_info: + used, matches = is_used_object(uci, f'users/{section}') + user['used'] = used + user['matches'] = matches + users.append(user) + return users + +def list_dhcp_static_leases(uci, expand=False, used_info=False): + """ + Get all DHCP static leases from dhcp config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + used_info: include used and matches info + + Returns: + a list of all DHCP static leases + """ + leases = [] + for section in uci.get_all("dhcp"): + lease = {} + if uci.get('dhcp', section) == 'host': + obj = uci.get_all('dhcp', section) + lease['id'] = f"dhcp/{section}" + lease['name'] = obj.get('name') + lease['type'] = 'dhcp_static_lease' + lease['subtype'] = 'dhcp_static_lease' + lease['family'] = 'ipv4' + if expand: + lease['ipaddr'] = [obj.get('ip')] + if used_info: + used, matches = is_used_object(uci, f'dhcp/{section}') + lease['used'] = used + lease['matches'] = matches + leases.append(lease) + return leases + +def list_dns_records(uci, expand=False, used_info=False): + """ + Get all DNS records from dhcp config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + used_info: include used and matches info + + Returns: + a list of all DNS records + """ + records = [] + for section in uci.get_all("dhcp"): + record = {} + if uci.get('dhcp', section) == 'domain': + obj = uci.get_all('dhcp', section) + record['id'] = f"dhcp/{section}" + record['name'] = obj.get('name') + record['type'] = 'dns_record' + record['subtype'] = 'dns_record' + record['family'] = 'ipv4' + if expand: + record['ipaddr'] = [obj.get('ip')] + if used_info: + used, matches = is_used_object(uci, f'dhcp/{section}') + record['used'] = used + record['matches'] = matches + records.append(record) + return records + +def list_objects(uci, include_domain_sets=True, include_host_sets=True, singleton_only=False, expand=False): + """ + Get objects from objects, dhcp, and users config + + Args: + uci: EUci pointer + include_domain_sets: include domain sets in the list + include_host_sets: include host sets in the list + expand: expand the list with all IP addresses of the object + + Returns: + a list of all objects + """ + hsets = [] + dsets = [] + if include_host_sets: + for h in list_host_sets(uci, True): + if singleton_only and not h['singleton']: + continue + h['id'] = f"objects/{h['id']}" + h['type'] = 'host_set' + if not expand: + del[h['ipaddr']] + hsets.append(h) + + if include_domain_sets: + for d in list_domain_sets(uci, True): + d['id'] = f"objects/{d['id']}" + d['type'] = 'domain_set' + d['subtype'] = 'domain_set' + if not expand: + del[d['domain']] + del[d['timeout']] + dsets.append(d) + vpn_users = list_vpn_users(uci, expand, True) + dhcp_static_leases = list_dhcp_static_leases(uci, expand, True) + dns_records = list_dns_records(uci, expand, True) + return hsets + dsets + vpn_users + dhcp_static_leases + dns_records + +def get_info(uci, database_id): + """ + Get the info of the object. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + a dictionary with the following fields: + + - `database`: the database of the object + - `id`: the id of the object + - `name`: the name of the object + - `type`: the type of the object + - `family`: IP family, like `ipv4` or `ipv6` + """ + try: + database, id = database_id.split('/') + type = uci.get(database, id) + name = uci.get(database, id, 'name', default=None) + family = uci.get(database, id, 'family', default='ipv4') + if not name: + name = uci.get(database, id, 'label', default=None) + if not name: + name = id + return {'database': database, 'id': id, 'name': name, 'type': type, 'family': family} + except: + return None + diff --git a/packages/python3-nethsec/src/nethsec/ovpn/__init__.py b/packages/python3-nethsec/src/nethsec/ovpn/__init__.py new file mode 100644 index 000000000..58658d257 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/ovpn/__init__.py @@ -0,0 +1,285 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Library for common VPN functions. +""" + +import json +import ipaddress +import random +import struct +import socket +import subprocess +from nethsec import utils + +def get_local_networks(u): + """ + Return a list of local networks. + + Arguments: + - u -- EUci instance + + Returns: + - a list of local networks + """ + ret = set() + for l in utils.get_all_lan_devices(u): + try: + data = json.loads(subprocess.run(["ip", "--json", "address", "show", "dev", l], capture_output=True, text=True, check=True).stdout) + if len(data) > 0: + for addr in data[0].get('addr_info', []): + if addr.get("local", None) and addr.get("family", None) == "inet": # ipv4 only + # skip bond management address + if addr.get("local").startswith("127"): + continue + net = ipaddress.ip_interface(f'{addr.get("local")}/{addr.get("prefixlen")}').network + ret.add(f'{net}') + except: + continue + return list(ret) + +def get_public_addresses(u): + """ + Return a list of public addresses. + + Arguments: + - u -- EUci instance + + Returns: + - a list of public addresses + """ + ret = {} + for w in utils.get_all_wan_devices(u): + try: + data = json.loads(subprocess.run(["ip", "--json", "address", "show", "dev", w], capture_output=True, text=True, check=True).stdout) + except: + continue + if len(data) > 0: + for addr in data[0].get('addr_info', []): + if addr.get("local", None): + try: + res = subprocess.run([ + "/usr/bin/dig", + "-b", + addr.get("local"), + "+short", + "+time=1", + "myip.opendns.com", + "@resolver1.opendns.com" + ], + capture_output=True, text=True, check=True) + ret[res.stdout.strip()] = 1 + except: + pass + return list(ret.keys()) + +def to_cidr(netmask): + """ + Convert a netmask to CIDR notation. + + Arguments: + - netmask -- netmask in dotted decimal notation + + Returns: + - CIDR notation + + """ + return sum([bin(int(x)).count('1') for x in netmask.split('.')]) + +def to_netmask(prefix): + """ + Convert a CIDR notation to netmask. + + Arguments: + - prefix -- CIDR notation + + Returns: + - netmask in dotted decimal notation + """ + return socket.inet_ntoa(struct.pack(">I", (0xffffffff << (32 - int(prefix))) & 0xffffffff)) + +def generate_random_network(u): + """ + Generate a random network. + + Arguments: + - u -- EUci instance + + Returns: + - a random unused network + """ + network = random_network() + while is_used_network(u, network): + network = random_network() + return network + +def random_network(): + """ + Generate a random private network address. + + Returns: + - a random private network address + """ + return f"10.{random.randint(0, 254)}.{random.randint(0, 254)}.0/24" + +def is_used_network(u, network): + """ + Check if a network is already used by another OpenVPN. + + Arguments: + - u -- EUci instance + - network -- network to check + + Returns: + - True if the network is already used, False otherwise + """ + try: + openvpn = u.get_all('openvpn') + except: + return False + + for v in openvpn: + ifconfig = u.get('openvpn', v, 'ifconfig', default="") + if ifconfig: + ifnet = f'{ifconfig.split(" ")[0]}/24' # assume /24 + if ipaddress.ip_network(ifnet, strict=False) == ipaddress.ip_network(network): + return True + + server = opt2cidr(u.get('openvpn', v, 'server', default="")) + if server and server == network: + return True + + server_bridge = opt2cidr(u.get('openvpn', v, 'server_bridge', default="")) + if server_bridge: + if ipaddress.ip_network(server_bridge, strict=False) == ipaddress.ip_network(network): + return True + return False + +def opt2cidr(opt): + """ + Convert an OpenVPN option to CIDR notation. + + Arguments: + - opt -- OpenVPN option + + Returns: + - CIDR notation + """ + try: + tmp = opt.split(" ") + return f'{tmp[0]}/{to_cidr(tmp[1])}' + except: + return "" + + +def list_cipher(): + """ + Return a list of OpenVPN ciphers. + + Returns: + - a list of OpenVPN ciphers, each element is a dicttionary with the following keys: + - name: cipher name + - description: cipher description (weak or strong) + """ + ret = [] + try: + result = subprocess.run(['/usr/sbin/openvpn', '--show-ciphers'], capture_output=True, text=True, check=True) + output_lines = result.stdout.splitlines() + + for line in output_lines: + if '(' in line: + description = 'weak' + tmp = line.split() + cipher_name = tmp[0] + try: + bits = int(cipher_name.split("-")[1]) + except: + bits = 0 + if bits in [192, 256, 384, 512]: + description = 'strong' + ret.append({'name': cipher_name, 'description': description}) + except: + return {"ciphers": []} + return {"ciphers": ret} + +def list_digest(): + """ + Return a list of OpenVPN digests. + + Returns: + - a list of OpenVPN digests, each element is a dicttionary with the following keys: + - name: digest name + - description: digest description (weak or strong) + """ + ret = [] + try: + result = subprocess.run(['/usr/sbin/openvpn', '--show-digests'], capture_output=True, text=True, check=True) + output_lines = result.stdout.splitlines() + + for line in output_lines: + if 'bit' in line: + description = 'weak' + tmp = line.split() + if tmp[1] in ['224', '256', '384', '512', 'whirlpool']: + description = 'strong' + ret.append({'name': tmp[0], 'description': description}) + except: + return {"digests": []} + return {"digests": ret} + +def generate_random_port(uci, limit_min, limit_max): + ''' + Generate a random port. + + Arguments: + - uci -- EUci instance + - limit_min -- minimum port number + - limit_max -- maximum port number + + Returns: + - a random port number + ''' + port = random.randint(limit_min, limit_max) + while is_used_port(uci, port): + port = random.randint(limit_min, limit_max) + return port + +def is_used_port(uci, port): + ''' + Check if a port is already used by another OpenVPN. + + Arguments: + - uci -- EUci instance + - port -- port to check + + Returns: + - True if the port is already used, False otherwise + ''' + for v in uci.get_all('openvpn'): + rport = int(uci.get('openvpn', v, 'port', default="0")) + lport = int(uci.get('openvpn', v, 'lport', default="0")) + if port == rport or port == lport: + return True + return False + +def list_connected_clients(openvpninstance, type="subnet"): + """ + List connected clients. + + Arguments: + - openvpninstance -- OpenVPN instance name + - type -- type of list (subnet or p2p) + + Returns: + - a list of connected clients + """ + try: + p = subprocess.run(["/usr/bin/openvpn-status", "-t", type, f"/var/run/openvpn_{openvpninstance}.socket"], check=True, capture_output=True, text=True) + except: + return {} + return json.loads(p.stdout) diff --git a/packages/python3-nethsec/src/nethsec/reverse_proxy/__init__.py b/packages/python3-nethsec/src/nethsec/reverse_proxy/__init__.py new file mode 100644 index 000000000..3e239b1de --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/reverse_proxy/__init__.py @@ -0,0 +1,249 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Reverse proxy helper functions + +All the functions in this module are meant to be used by the reverse proxy api and migration script. +Be careful when using them in other contexts. +""" + +import ipaddress +import os +import subprocess + +from euci import EUci + +from nethsec import utils + + +def set_proxy_pass(e_uci, location, proxy_pass): + """ + Set the proxy pass for the specified location. + This function is a mere workaround for nginx to allow non-resolvable domains to be used as proxy_pass. + + Args: + - e_uci: EUci object + - location: nginx config location id + - proxy_pass: string of the proxy pass to set + """ + prefix = proxy_pass.split('://')[0] + destination = proxy_pass.split('://')[1] + address = destination.split('/')[0] + if len(destination.split('/')) > 1: + path = '/' + '/'.join(destination.split('/')[1:]) + else: + path = '' + + try: + ipaddress.ip_address(address) + e_uci.set('nginx', location, 'proxy_pass', proxy_pass) + except ValueError: + e_uci.set('nginx', location, 'resolver', '127.0.0.1') + e_uci.set('nginx', location, 'set', [f'$upstream {address}']) + e_uci.set('nginx', location, 'proxy_pass', f'{prefix}://$upstream{path}') + + +def create_location(e_uci, uci_server, location, proxy_pass, domain='', allow=None): + """ + Create a new location in the nginx config. + + Args: + - e_uci: EUci object + - uci_server: where the location is defined + - location: path where the location will answer + - proxy_pass: where the location will proxy to + - domain: optional domain to set in the Host header + - allow: array of allowed ip addresses + + Returns: + location id of the created location + """ + if allow is None: + allow = [] + location_id = utils.get_random_id() + e_uci.set('nginx', location_id, 'location') + # defaults + e_uci.set('nginx', location_id, 'proxy_http_version', '1.1') + default_headers = [ + 'X-Forwarded-For $proxy_add_x_forwarded_for', + 'X-Real-IP $remote_addr', + 'Upgrade $http_upgrade', + 'Connection "upgrade"' + ] + if domain: + default_headers.append(f'Host {domain}') + e_uci.set('nginx', location_id, 'proxy_set_header', default_headers) + # set body limit to 1GB: same value of NethServer 7 + e_uci.set('nginx', location_id, 'client_max_body_size', '1024m') + # setup location + e_uci.set('nginx', location_id, 'uci_server', uci_server) + e_uci.set('nginx', location_id, 'location', location) + if len(allow) > 0: + e_uci.set('nginx', location_id, 'allow', allow) + set_proxy_pass(e_uci, location_id, proxy_pass) + + return location_id + + +def add_path(path, destination, description, allow): + """ + Add a new location to the _lan server. + + Args: + - path: path where the location will answer + - destination: destination endpoint + - description: description of the location + - allow: array of allowed ip addresses + """ + e_uci = EUci() + location = create_location(e_uci, '_lan', path, destination, allow=allow) + e_uci.set('nginx', location, 'uci_description', description) + # defaults + e_uci.set('nginx', location, 'proxy_ssl_verify', 'off') + # add location to _lan server + includes = list(e_uci.get('nginx', '_lan', 'include', list=True)) + if 'conf.d/_lan[.]proxy' not in includes: + includes.append('conf.d/_lan[.]proxy') + e_uci.set('nginx', '_lan', 'include', includes) + + e_uci.save('nginx') + + +def add_domain(domain, destination, certificate, description, allow): + """ + Add a new server to the nginx config. + + Args: + - domain: domain to answer to + - destination: where to proxy the request to + - certificate: certificate to be used, beware the certificate must be in the list of certificates + - description: easy description of the server + - allow: ip addresses allowed to access the server + """ + e_uci = EUci() + # create server instance + server_name = utils.get_random_id() + e_uci.set('nginx', server_name, 'server') + # create default location + create_location(e_uci, server_name, '/', destination, domain, allow) + # defaults + e_uci.set('nginx', server_name, 'proxy_ssl_verify', 'off') + e_uci.set('nginx', server_name, 'ssl_session_timeout', '64m') + e_uci.set('nginx', server_name, 'ssl_session_cache', 'shared:SSL:32k') + e_uci.set('nginx', server_name, 'listen', ['443 ssl', '[::]:443 ssl']) + e_uci.set('nginx', server_name, 'include', [f'conf.d/{server_name}.proxy']) + e_uci.set('nginx', server_name, 'access_log', 'syslog:server=unix:/dev/log') + e_uci.set('nginx', server_name, 'error_log', 'syslog:server=unix:/dev/log') + # setup server + valid_certificates = {name: certificate for (name, certificate) in certificate_list(e_uci).items() + if 'cert_path' in certificate} + e_uci.set('nginx', server_name, 'server_name', domain) + e_uci.set('nginx', server_name, 'ssl_certificate', valid_certificates[certificate]['cert_path']) + e_uci.set('nginx', server_name, 'ssl_certificate_key', valid_certificates[certificate]['key_path']) + e_uci.set('nginx', server_name, 'uci_description', description) + + e_uci.save('nginx') + + +def certificate_list(uci: EUci): + """ + List all the certificates in the system. + + Args: + - uci: euci object + """ + default_certificate = uci.get('nginx', '_lan', 'ssl_certificate') + + servers = utils.get_all_by_type(uci, 'nginx', 'server') + certificate_domain = {} + for server in servers: + if 'ssl_certificate' in servers[server]: + if server not in certificate_domain: + certificate_domain[servers[server]['ssl_certificate']] = [] + certificate_domain[servers[server]['ssl_certificate']].append(servers[server]['server_name']) + + certificates = {} + + # scan default certificates + for entry in os.scandir('/etc/nginx/conf.d'): + if entry.is_file() and entry.name.endswith('.crt') and os.path.isfile(entry.path[:-4] + '.key'): + certificates[entry.name[:-4]] = { + 'type': 'self-signed', + 'cert_path': entry.path, + 'key_path': entry.path[:-4] + '.key', + 'default': default_certificate == entry.path, + } + + # scan custom certificates + if not os.path.isdir('/etc/nginx/custom_certs'): + os.mkdir('/etc/nginx/custom_certs') + for entry in os.scandir('/etc/nginx/custom_certs'): + if entry.is_file() and entry.name.endswith('.crt') and os.path.isfile(entry.path[:-4] + '.key'): + certificates[entry.name[:-4]] = { + 'type': 'custom', + 'cert_path': entry.path, + 'key_path': entry.path[:-4] + '.key', + 'default': default_certificate == entry.path, + } + + # scan acme certificates + requested_certificates = utils.get_all_by_type(uci, 'acme', 'cert') + enabled_certificates = [certificate for certificate in requested_certificates + if requested_certificates[certificate]['enabled'] == '1'] + for certificate in enabled_certificates: + domain = requested_certificates[certificate]['domains'][0] + cert_path = f'/etc/ssl/acme/{domain}.fullchain.crt' + key_path = f'/etc/ssl/acme/{domain}.key' + if os.path.isfile(cert_path): + certificates[certificate] = { + 'type': 'acme', + 'pending': False, + 'requested_domains': requested_certificates[certificate]['domains'], + 'cert_path': cert_path, + 'key_path': key_path, + 'default': default_certificate == cert_path + } + else: + certificates[certificate] = { + 'type': 'acme', + 'pending': True, + 'requested_domains': requested_certificates[certificate]['domains'], + } + + for certificate in certificates: + if 'cert_path' in certificates[certificate]: + # get certificate details + details = subprocess.run( + ['openssl', 'x509', '-noout', '-text', '-in', certificates[certificate]['cert_path']], + check=True, capture_output=True, text=True) + # get certificate expiration + expiration = subprocess.run(['openssl', 'x509', '-noout', '-enddate', '-dateopt', 'iso_8601', '-in', + certificates[certificate]['cert_path']], + check=True, capture_output=True, text=True) + # fetch subject common name + subject = subprocess.run(['openssl', 'x509', '-noout', '-subject', '-nameopt', 'multiline', '-in', + certificates[certificate]['cert_path']], + check=True, capture_output=True, text=True) + cn = '' + for row in subject.stdout.split('\n'): + if 'commonName' in row: + cn = row.split('=')[1].strip() + break + + # get certificate assigned domains + servers = [] + if certificates[certificate]['cert_path'] in certificate_domain: + servers = certificate_domain[certificates[certificate]['cert_path']] + + # store certificate details + certificates[certificate]['details'] = details.stdout + certificates[certificate]['expiration'] = expiration.stdout.split('=')[1].strip() + certificates[certificate]['servers'] = servers + certificates[certificate]['domain'] = cn + + return certificates diff --git a/packages/python3-nethsec/src/nethsec/snort/__init__.py b/packages/python3-nethsec/src/nethsec/snort/__init__.py new file mode 100644 index 000000000..dc03ea423 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/snort/__init__.py @@ -0,0 +1,466 @@ +# Copyright (c) 2011 Jason Ish +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +""" Module for parsing Snort-like rules. + +Parsing is done using regular expressions and the job of this module +is to do its best at parsing out fields of interest from the rule +rather than perform a sanity check. + +The methods that parse multiple rules for a provided input +(parse_file, parse_fileobj) return a list of rules instead of dict +keyed by ID as its not the job of this module to detect or deal with +duplicate signature IDs. +""" + +import sys +import re +import logging +import io + +logger = logging.getLogger(__name__) + +# Compile an re pattern for basic rule matching. +rule_pattern = re.compile(r"^(?P#)*[\s#]*" + r"(?P" + r"(?P
[^()]+)" + r"\((?P.*)\)" + r"$)") + +# Rule actions we expect to see. +actions = ( + "alert", "config", "log", "pass", "activate", "dynamic", "drop", "reject", "sdrop") + +class Rule(dict): + """Class representing a rule. + + The Rule class is a class that also acts like a dictionary. + + Dictionary fields: + + - **group**: The group the rule belongs to, typically the filename. + - **enabled**: True if rule is enabled (uncommented), False is + disabled (commented) + - **action**: The action of the rule (alert, pass, etc) as a + string + - **proto**: The protocol string of the rule. + - **source_addr**: The source address string of the rule. + - **source_port**: The source ports string of the rule. + - **direction**: The direction string of the rule. + - **dest_addr**: The destination address string of the rule. + - **dest_port**: The destination ports string of the rule. + - **gid**: The gid of the rule as an integer + - **sid**: The sid of the rule as an integer + - **rev**: The revision of the rule as an integer + - **msg**: The rule message as a string + - **flowbits**: List of flowbit options in the rule + - **metadata**: Metadata values as a list + - **references**: References as a list + - **classtype**: The classification type + - **priority**: The rule priority, 0 if not provided + - **raw**: The raw rule as read from the file or buffer + + :param enabled: Optional parameter to set the enabled state of the rule + :param action: Optional parameter to set the action of the rule + :param group: Optional parameter to set the group (filename) of the rule + + """ + def __init__(self, enabled=None, action=None, group=None): + dict.__init__(self) + self["enabled"] = enabled + self["action"] = action + self["proto"] = None + self["source_addr"] = None + self["source_port"] = None + self["direction"] = None + self["dest_addr"] = None + self["dest_port"] = None + self["group"] = group + self["gid"] = 1 + self["sid"] = None + self["rev"] = None + self["msg"] = None + self["flowbits"] = [] + self["metadata"] = [] + self["references"] = [] + self["classtype"] = None + self["priority"] = 0 + + self["options"] = [] + + self["raw"] = None + + def __getattr__(self, name): + return self[name] + + @property + def id(self): + """ The ID of the rule. + + :returns: A tuple (gid, sid) representing the ID of the rule + :rtype: A tuple of 2 ints + """ + return (int(self.gid), int(self.sid)) + + @property + def idstr(self): + """Return the gid and sid of the rule as a string formatted like: + '[GID:SID]'""" + return "[%s:%s]" % (str(self.gid), str(self.sid)) + + def brief(self): + """ A brief description of the rule. + + :returns: A brief description of the rule + :rtype: string + """ + return "%s[%d:%d] %s" % ( + "" if self.enabled else "# ", self.gid, self.sid, self.msg) + + def __hash__(self): + return self["raw"].__hash__() + + def __str__(self): + """ The string representation of the rule. + + If the rule is disabled it will be returned as commented out. + """ + return self.format() + + def format(self): + return u"%s%s" % (u"" if self.enabled else u"# ", self.raw) + + def rebuild_options(self): + """ Rebuild the rule options from the list of options.""" + options = [] + for option in self.options: + if option["value"] is None: + options.append(option["name"]) + else: + options.append("%s:%s" % (option["name"], option["value"])) + return "%s;" % "; ".join(options) + +def remove_option(rule, name): + rule["options"] = [ + option for option in rule["options"] if option["name"] != name] + new_rule_string = "%s%s (%s)" % ( + "" if rule.enabled else "# ", + rule["header"].strip(), + rule.rebuild_options()); + return parse(new_rule_string, rule["group"]) + +def add_option(rule, name, value, index=None): + option = { + "name": name, + "value": value, + } + if index is None: + rule["options"].append(option) + else: + rule["options"].insert(index, option) + new_rule_string = "%s%s (%s)" % ( + "" if rule.enabled else "# ", + rule["header"].strip(), + rule.rebuild_options()) + return parse(new_rule_string, rule["group"]) + +def find_opt_end(options): + """ Find the end of an option (;) handling escapes. """ + offset = 0 + + while True: + i = options[offset:].find(";") + if options[offset + i - 1] == "\\": + offset += 2 + else: + return offset + i + +def parse(buf, group=None): + """ Parse a single rule for a string buffer. + + :param buf: A string buffer containing a single Snort-like rule + + :returns: An instance of of :py:class:`.Rule` representing the parsed rule + """ + + if type(buf) == type(b""): + buf = buf.decode("utf-8") + buf = buf.strip() + + m = rule_pattern.match(buf) + if not m: + return None + + if m.group("enabled") == "#": + enabled = False + else: + enabled = True + + header = m.group("header").strip() + + # If a decoder rule, the header will be one word. + if len(header.split(" ")) == 1: + action = header + proto = None + source_addr = None + source_port = None + direction = None + dest_addr = None + dest_port = None + else: + states = ["action", + "proto", + "source_addr", + "source_port", + "direction", + "dest_addr", + "dest_port", + ] + state = 0 + + rem = header + while state < len(states): + if not rem: + return None + if rem[0] == "[": + end = rem.find("]") + if end < 0: + return + end += 1 + token = rem[:end].strip() + rem = rem[end:].strip() + else: + end = rem.find(" ") + if end < 0: + token = rem + rem = "" + else: + token = rem[:end].strip() + rem = rem[end:].strip() + + if states[state] == "action": + action = token + elif states[state] == "proto": + proto = token + elif states[state] == "source_addr": + source_addr = token + elif states[state] == "source_port": + source_port = token + elif states[state] == "direction": + direction = token + elif states[state] == "dest_addr": + dest_addr = token + elif states[state] == "dest_port": + dest_port = token + + state += 1 + + if action not in actions: + return None + + rule = Rule(enabled=enabled, action=action, group=group) + rule["header"] = header + rule["proto"] = proto + rule["source_addr"] = source_addr + rule["source_port"] = source_port + rule["direction"] = direction + rule["dest_addr"] = dest_addr + rule["dest_port"] = dest_port + + options = m.group("options") + + while True: + if not options: + break + index = find_opt_end(options) + if index < 0: + raise Exception("end of option not found: %s" % (buf)) + option = options[:index].strip() + options = options[index + 1:].strip() + + if option.find(":") > -1: + name, val = [x.strip() for x in option.split(":", 1)] + else: + name = option + val = None + + rule["options"].append({ + "name": name, + "value": val, + }) + + if name in ["gid", "sid", "rev"]: + rule[name] = int(val) + elif name == "metadata": + if not name in rule: + rule[name] = [] + rule[name] += [v.strip() for v in val.split(",")] + elif name == "flowbits": + rule.flowbits.append(val) + elif name == "reference": + rule.references.append(val) + elif name == "msg": + if val.startswith('"') and val.endswith('"'): + val = val[1:-1] + rule[name] = val + else: + rule[name] = val + + if rule["msg"] is None: + rule["msg"] = "" + + rule["raw"] = m.group("raw").strip() + + return rule + +def parse_fileobj(fileobj, group=None): + """ Parse multiple rules from a file like object. + + Note: At this time rules must exist on one line. + + :param fileobj: A file like object to parse rules from. + + :returns: A list of :py:class:`.Rule` instances, one for each rule parsed + """ + rules = [] + buf = "" + for line in fileobj: + try: + if type(line) == type(b""): + line = line.decode() + except: + pass + if line.rstrip().endswith("\\"): + buf = "%s%s " % (buf, line.rstrip()[0:-1]) + continue + try: + rule = parse(buf + line, group) + if rule: + rules.append(rule) + except: + logger.error("failed to parse rule: %s" % (buf)) + raise + buf = "" + return rules + +def parse_file(filename, group=None): + """ Parse multiple rules from the provided filename. + + :param filename: Name of file to parse rules from + + :returns: A list of :py:class:`.Rule` instances, one for each rule parsed + """ + with io.open(filename, encoding="utf-8") as fileobj: + return parse_fileobj(fileobj, group) + +class FlowbitResolver(object): + + setters = ["set", "setx", "unset", "toggle"] + getters = ["isset", "isnotset"] + + def __init__(self): + self.enabled = [] + + def resolve(self, rules): + required = self.get_required_flowbits(rules) + enabled = self.set_required_flowbits(rules, required) + if enabled: + self.enabled += enabled + return self.resolve(rules) + return self.enabled + + def set_required_flowbits(self, rules, required): + enabled = [] + for rule in [rule for rule in rules.values() if not rule.enabled]: + for option, value in map(self.parse_flowbit, rule.flowbits): + if option in self.setters and value in required: + rule.enabled = True + enabled.append(rule) + return enabled + + def get_required_rules(self, rulemap, flowbits, include_enabled=False): + """Returns a list of rules that need to be enabled in order to satisfy + the list of required flowbits. + + """ + required = [] + + for rule in [rule for rule in rulemap.values()]: + if not rule: + continue + for option, value in map(self.parse_flowbit, rule.flowbits): + if option in self.setters and value in flowbits: + if rule.enabled and not include_enabled: + continue + required.append(rule) + + return required + + def get_required_flowbits(self, rules): + required_flowbits = set() + for rule in [rule for rule in rules.values() if rule and rule.enabled]: + for option, value in map(self.parse_flowbit, rule.flowbits): + if option in self.getters: + required_flowbits.add(value) + return required_flowbits + + def parse_flowbit(self, flowbit): + tokens = flowbit.split(",", 1) + if len(tokens) == 1: + return tokens[0], None + elif len(tokens) == 2: + return tokens[0], tokens[1] + else: + raise Exception("Flowbit parse error on %s" % (flowbit)) + +def enable_flowbit_dependencies(rulemap): + """Helper function to resolve flowbits, wrapping the FlowbitResolver + class. """ + resolver = FlowbitResolver() + return resolver.resolve(rulemap) + +def format_sidmsgmap(rule): + """ Format a rule as a sid-msg.map entry. """ + try: + return " || ".join([str(rule.sid), rule.msg] + rule.references) + except: + logger.error("Failed to format rule as sid-msg.map: %s" % (str(rule))) + return None + +def format_sidmsgmap_v2(rule): + """ Format a rule as a v2 sid-msg.map entry. + + eg: + gid || sid || rev || classification || priority || msg || ref0 || refN + """ + try: + return " || ".join([ + str(rule.gid), str(rule.sid), str(rule.rev), + "NOCLASS" if rule.classtype is None else rule.classtype, + str(rule.priority), rule.msg] + rule.references) + except: + logger.error("Failed to format rule as sid-msg-v2.map: %s" % ( + str(rule))) + return None diff --git a/packages/python3-nethsec/src/nethsec/users/__init__.py b/packages/python3-nethsec/src/nethsec/users/__init__.py new file mode 100644 index 000000000..429ae3d36 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/users/__init__.py @@ -0,0 +1,932 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Library that handles users and groups +""" + +import base64 +import hashlib +import ipaddress +import json +import os +import subprocess +import secrets + +from euci import EUci +from uci import UciExceptionNotFound + +from nethsec import utils +from nethsec.ldif import LDIFParser +from urllib.parse import urlparse +from passlib import hash +from io import BytesIO + + +def get_database_type(uci, database): + ''' + Retrieve database type + + Arguments: + - uci -- EUci pointer + - database -- Database identifier + + Returns: + - Database type (local or ldap) + ''' + return uci.get('users', database, default='') + +def get_user_by_name(uci, name, database="main"): + ''' + Retrieve a user by name + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Local database identifier (default: main) + + Returns: + - A user object or None if not found + ''' + user = {"local": get_database_type(uci, database) == "local"} + for u in uci.get_all('users'): + if uci.get('users', u, 'database', default='') != database: + continue + if uci.get('users', u, default='') == "user": + if uci.get('users', u, 'name', default='').lower() == name.lower(): + for opt in uci.get_all("users", u): + user[opt] = uci.get_all('users', u, opt) + # convert tuple to list + if type(user[opt]) is tuple: + user[opt] = list(user[opt]) + if user["local"]: + user["admin"] = is_admin(uci, name) + else: + user["admin"] = False + user['id'] = u + return user + return None + +def get_group_by_name(uci, name, database="main"): + ''' + Retrieve a group by name + + Arguments: + - uci -- EUci pointer + - name -- Group name + - database -- Local database identifier (default: main) + + + Returns: + - A group object or None if not found + ''' + group = {"local": get_database_type(uci, database) == "local"} + for g in uci.get_all('users'): + if uci.get('users', g, 'database', default='') != database: + continue + if uci.get('users', g, default='') == "group": + if uci.get('users', g, 'name', default='') == name: + for opt in uci.get_all("users", g): + group[opt] = uci.get_all('users', g, opt) + # convert tuple to list + if type(group[opt]) is tuple: + group[opt] = list(group[opt]) + group['id'] = g + return group + return None + +def get_user_addresses(uci, user): + ''' + Retrieve all IP addresses associated to given user + + Arguments: + - uci -- EUci pointer + - user -- User object id (UCI section) + + Returns a tuple of lists: + - first element is a list of IPv4 addresses + - second element is a list of IPv6 addresses + ''' + ipv4 = [] + ipv6 = [] + user_obj = get_user_by_name(uci, user) + # get addresses from plain ipaddr option + for ip in uci.get('users', user_obj['id'], 'ipaddr', list=True, default=[]): + if ':' in ip: + ipv6.append(ip) + else: + ipv4.append(ip) + # get vpn reservation + ovpn_ipaddr = uci.get('users', user_obj['id'], 'openvpn_ipaddr', default="") + if ovpn_ipaddr: + if ':' in ovpn_ipaddr: + ipv6.append(ovpn_ipaddr) + else: + ipv4.append(ovpn_ipaddr) + # get address from DNS record and DHCP reservation + for st in ['host', 'domain']: + for h in uci.get('objects', user_obj['id'], st, list=True, default=[]): # dhcp reservation + ip = uci.get('dhcp', h, 'ip', default='') + if not ip: + continue + if ':' in ip: + ipv6.append(ip) + else: + ipv4.append(ip) + + return (ipv4, ipv6) + +def get_user_macs(uci, user): + ''' + Retrieve all MAC addresses associated to given user + + Arguments: + - uci -- EUci pointer + - user -- User object id (UCI section) + + Returns: + - A list of MAC addresses + ''' + user_obj = get_user_by_name(uci, user) + return list(uci.get('users', user_obj['id'], 'macaddr', list=True, default=[])) + +def get_group_addresses(uci, group): + ''' + Retrieve all IP addresses associated to given group + + Arguments: + - uci -- EUci pointer + - user -- Group object id (UCI section) + + Returns: + - A tuple of lists: + - first element is a list of IPv4 addresses + - second element is a list of IPv6 addresses + ''' + ipv4 = [] + ipv6 = [] + group_obj = get_group_by_name(uci, group) + for u in uci.get('users', group_obj['id'], 'user', list=True, default=[]): + (uipv4, uipv6) = get_user_addresses(uci, u) + ipv4 = ipv4 + uipv4 + ipv6 = ipv6 + uipv6 + return (ipv4, ipv6) + +def get_group_macs(uci, group): + ''' + Retrieve all MAC addresses associated to given group + + Arguments: + - uci -- EUci pointer + - group -- Group object id (UCI section) + + Returns: + - A list of MAC addresses + ''' + macs = [] + group_obj = get_group_by_name(uci, group) + for u in uci.get('users', group_obj['id'], 'user', list=True, default=[]): + macs = macs + get_user_macs(uci, u) + return macs + +def list_users(uci, database='main'): + ''' + Retrieve all users + + Arguments: + - database -- Database identifier (default: main) + + Returns: + - A list of user objects + ''' + users = [] + try: + dbconf = uci.get_all('users', database) + except: + raise utils.ValidationError('database', 'db_not_found', database) + dbtype = get_database_type(uci, database) + if dbtype == "local": + for u in utils.get_all_by_type(uci, 'users', 'user'): + if uci.get('users', u, 'database', default='') != database: + continue + if uci.get('users', u, default='') == "user": + username = uci.get('users', u, 'name', default='') + user = get_user_by_name(uci, username, database) + users.append(user) + users = sorted(users, key=lambda u: u['name']) + elif dbtype == "ldap": + # retrieve also user_cn attribute for old configurations + display_attr = dbconf.get('user_display_attr', dbconf.get('user_cn', 'cn')) + users = list_remote_users(dbconf.get('uri'), dbconf.get('user_dn'), dbconf.get('user_attr'), display_attr, dbconf.get('start_tls') == '1', dbconf.get('tls_reqcert'), dbconf.get('bind_dn'), dbconf.get('bind_password'), dbconf.get('schema')) + for u in users: + user = get_user_by_name(uci, u['name'], database) + if user: + u.update(user) + else: + u['id'] = None + u['local'] = False + u['admin'] = False + u['database'] = database + pass + + return users + +def add_ldap_database(uci, name, uri, schema, base_dn, user_dn, user_attr, user_display_attr, start_tls=False, tls_reqcert='never', description="", bind_dn=None, bind_password=None, user_bind_dn=None): + ''' + Add a new LDAP database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + - uri -- LDAP URI + - schema -- LDAP schema + - base_dn -- LDAP base DN + - user_dn -- LDAP user DN + - user_attr -- LDAP user attribute + - user_display_attr -- LDAP user full name attribute + - start_tls -- Use TLS (default: False) + - tls_reqcert -- TLS certificate validation (default: never) + - description -- Database description (default: "") + - bind_dn -- LDAP bind DN + - bind_password -- LDAP bind password + - user_bind_dn -- LDAP custom user bind DN + + Returns: + - The database identifier + ''' + if uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_already_exists', name) + ldap = uci.set('users', name, 'ldap') + uci.set('users', name, 'uri', uri) + uci.set('users', name, 'schema', schema) + uci.set('users', name, 'base_dn', base_dn) + uci.set('users', name, 'user_dn', user_dn) + uci.set('users', name, 'user_attr', user_attr) + uci.set('users', name, 'user_display_attr', user_display_attr) + uci.set('users', name, 'start_tls', '1' if start_tls else '0') + uci.set('users', name, 'tls_reqcert', tls_reqcert) + uci.set('users', name, 'description', description) + if bind_dn and bind_password: + uci.set('users', name, 'bind_dn', bind_dn) + uci.set('users', name, 'bind_password', bind_password) + if user_bind_dn: + uci.set('users', name, 'user_bind_dn', user_bind_dn) + uci.save("users") + return ldap + +def edit_ldap_database(uci, name, uri, schema, base_dn, user_dn, user_attr, user_display_attr, start_tls=False, tls_reqcert='never', description="", bind_dn=None, bind_password=None, user_bind_dn=None): + ''' + Edit an existing LDAP database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + - uri -- LDAP URI + - schema -- LDAP schema + - base_dn -- LDAP base DN + - user_dn -- LDAP user DN + - user_attr -- LDAP user attribute + - user_display_attr -- LDAP user full name attribute + - start_tls -- Use TLS (default: False) + - tls_reqcert -- TLS certificate validation (default: never) + - description -- Database description (default: "") + - bind_dn -- LDAP bind DN + - bind_password -- LDAP bind password + - user_bind_dn -- LDAP custom user bind DN + + Returns: + - The database identifier + ''' + if not uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_not_found', name) + if uci.get('users', name, default='') != "ldap": + raise utils.ValidationError('name', 'db_not_ldap', name) + uci.set('users', name, 'uri', uri) + uci.set('users', name, 'schema', schema) + uci.set('users', name, 'base_dn', base_dn) + uci.set('users', name, 'user_dn', user_dn) + uci.set('users', name, 'user_attr', user_attr) + uci.set('users', name, 'user_display_attr', user_display_attr) + uci.set('users', name, 'start_tls', '1' if start_tls else '0') + uci.set('users', name, 'tls_reqcert', tls_reqcert) + uci.set('users', name, 'description', description) + if bind_dn and bind_password: + uci.set('users', name, 'bind_dn', bind_dn) + uci.set('users', name, 'bind_password', bind_password) + else: + try: + uci.delete('users', name, 'bind_dn') + uci.delete('users', name, 'bind_password') + except: + pass + if user_bind_dn: + uci.set('users', name, 'user_bind_dn', user_bind_dn) + else: + try: + uci.delete('users', name, 'user_bind_dn') + except: + pass + # remove old unused user_cn field, if present + try: + uci.delete('users', name, 'user_cn') + except: + pass + uci.save("users") + return True + +def delete_ldap_database(uci, name): + ''' + Delete an existing LDAP database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + + Returns: + - True if successful + ''' + if not uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_not_found', name) + if uci.get('users', name, default='') != "ldap": + raise utils.ValidationError('name', 'db_not_ldap', name) + for u in uci.get_all('users'): + if uci.get('users', u, 'database', default='') == name: + uci.delete('users', u) + uci.delete('users', name) + uci.save("users") + return True + +def add_local_database(uci, name, description=""): + ''' + Add a new local database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + - description -- Database description (default: "") + + Returns: + - The database identifier + ''' + if uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_already_exists', name) + local = uci.set('users', name, 'local') + uci.set('users', name, 'description', description) + uci.save("users") + return True + +def edit_local_database(uci, name, description=""): + ''' + Edit an existing local database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + - description -- Database description (default: "") + + Returns: + - The database identifier + ''' + if not uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_not_found', name) + if uci.get('users', name, default='') != "local": + raise utils.ValidationError('name', 'db_not_local', name) + uci.set('users', name, 'description', description) + uci.save("users") + return True + +def delete_local_database(uci, name): + ''' + Delete an existing local database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + + Returns: + - True if successful + ''' + if not uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_not_found', name) + if uci.get('users', name, default='') != "local": + raise utils.ValidationError('name', 'db_not_local', name) + uci.delete('users', name) + for u in uci.get_all('users'): + if uci.get('users', u, 'database', default='') == name: + uci.delete('users', u) + uci.save("users") + return True + +def list_databases(uci): + ''' + Retrieve all databases + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of database objects, each one containing: + - name: database identifier + - type: database type (local or ldap) + - description: database description + ''' + ret = [] + for db in uci.get_all('users'): + if uci.get('users', db, default='') == "local": + ret.append({"name": db, "type": "local", "description": uci.get('users', db, 'description', default='')}) + elif uci.get('users', db, default='') == "ldap": + ret.append({"name": db, "type": "ldap", "description": uci.get('users', db, 'description', default=''), + "schema": uci.get('users', db, 'schema', default=''), + "uri": uci.get('users', db, 'uri', default=''), + "used": used_by(uci, db)}) + return ret + +def get_database(uci, name): + ''' + Retrieve a database by name + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + + Returns: + - A database object or None if not found + ''' + try: + db = uci.get_all('users', name) + except: + return None + db["name"] = name + db["type"] = get_database_type(uci, name) + if 'user_cn' in db: + # migrate from old to new attribute names + db['user_display_attr'] = db['user_cn'] + return db + +def add_local_user(uci, name, password="", description="", database="main", extra_fields={}): + ''' + Add a new local user + + Arguments: + - uci -- EUci pointer + - name -- User name + - password -- User password + - description -- User description (default: "") + - database -- Local database identifier (default: main) + - extra_fields -- Extra fields to add to the user (default: {}) + + Returns: + - The user identifier + ''' + if get_user_by_name(uci, name, database): + raise utils.ValidationError('name', 'user_already_exists', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + id = utils.get_random_id() + user = uci.set('users', id, 'user') + uci.set('users', id, 'database', database) + uci.set('users', id, 'name', name) + if password: + uci.set('users', id, 'password', shadow_password(password)) + uci.set('users', id, 'description', description) + for key in extra_fields: + uci.set('users', id, key, extra_fields[key]) + uci.save("users") + return id + +def edit_local_user(uci, name, password="", description=None, database="main", extra_fields={}): + ''' + Edit an existing local user + + Arguments: + - uci -- EUci pointer + - name -- User name + - password -- User password + - description -- User description (default: None) + - database -- Local database identifier (default: main) + - extra_fields -- Extra fields to add to the user (default: {}) + + Returns: + - The user identifier + ''' + user = get_user_by_name(uci, name, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + if password: + shadow = shadow_password(password) + uci.set('users', user["id"], 'password', shadow) + # update password inside the rpcd configuration database + if is_admin(uci, name): + for l in utils.get_all_by_type(uci, 'rpcd', 'login'): + if uci.get('rpcd', l, 'username', default='') == name: + uci.set('rpcd', l, 'password', shadow) + uci.save("rpcd") + if description is not None: + uci.set('users', user["id"], 'description', description) + for key in uci.get_all('users', user["id"]): + if not key in ["name", "description", "password", "database"]: + uci.delete('users', user["id"], key) + for key in extra_fields: + uci.set('users', user["id"], key, extra_fields[key]) + uci.save("users") + return user["id"] + +def delete_local_user(uci, name, database="main"): + ''' + Delete an existing local user + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Local database identifier (default: main) + + Returns: + - True if successful + ''' + user = get_user_by_name(uci, name, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + # remove user from all groups + for g in uci.get_all('users'): + if uci.get('users', g, 'database', default='') != database: + continue + if uci.get('users', g, default='') == "group": + gusers = list(uci.get('users', g, 'user', list=True, default=[])) + if name in gusers: + gusers.remove(name) + uci.set('users', g, 'user', gusers) # the user field may be deleted by uci if the list is empty + uci.delete('users', user["id"]) + uci.save("users") + if is_admin(uci, name): + remove_admin(uci, name) + return True + +def add_local_group(uci, name, users=[], description="", database="main"): + ''' + Add a new local group + + Arguments: + - uci -- EUci pointer + - name -- Group name + - users -- List of users (default: []) + - description -- Group description (default: "") + - database -- Local database identifier (default: main) + + Returns: + - The group identifier + ''' + if get_group_by_name(uci, name, database): + raise utils.ValidationError('name', 'group_already_exists', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + + id = utils.get_random_id() + group = uci.set('users', id, 'group') + uci.set('users', id, 'database', database) + uci.set('users', id, 'name', name) + uci.set('users', id, 'description', description) + uci.set('users', id, 'user', users) + uci.save("users") + return id + +def edit_local_group(uci, name, users=[], description="", database="main"): + ''' + Edit an existing local group + + Arguments: + - uci -- EUci pointer + - name -- Group name + - users -- List of users (default: []) + - description -- Group description (default: "") + - database -- Local database identifier (default: main) + + Returns: + - The group identifier + ''' + group = get_group_by_name(uci, name, database) + if not group: + raise utils.ValidationError('name', 'group_not_found', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + uci.set('users', group["id"], 'user', users) + uci.set('users', group["id"], 'description', description) + uci.save("users") + return group["id"] + +def delete_local_group(uci, name, database="main"): + ''' + Delete an existing local group + + Arguments: + - uci -- EUci pointer + - name -- Group name + - database -- Local database identifier (default: main) + + Returns: + - True if successful + ''' + group = get_group_by_name(uci, name, database) + if not group: + raise utils.ValidationError('name', 'group_not_found', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + uci.delete('users', group["id"]) + uci.save("users") + return True + +def add_remote_user(uci, name, database, extra_fields={}): + ''' + Add a new remote user + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Database identifier + - extra_fields -- Extra fields to add to the user (default: {}) + + Returns: + - The user identifier + ''' + if get_user_by_name(uci, name, database): + raise utils.ValidationError('name', 'user_already_exists', name) + if get_database_type(uci, database) != "ldap": + + raise utils.ValidationError('database', 'db_not_ldap', database) + id = utils.get_random_id() + user = uci.set('users', id, 'user') + uci.set('users', id, 'database', database) + uci.set('users', id, 'name', name) + for key in extra_fields: + uci.set('users', id, key, extra_fields[key]) + uci.save("users") + return id + +def edit_remote_user(uci, name, database, extra_fields={}): + ''' + Edit an existing remote user + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Database identifier + - extra_fields -- Extra fields to add to the user (default: {}) + + Returns: + - The user identifier + ''' + user = get_user_by_name(uci, name, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', name) + if get_database_type(uci, database) != "ldap": + raise utils.ValidationError('database', 'db_not_ldap', database) + for key in uci.get_all('users', user["id"]): + if not key in ["name", "database"]: + uci.delete('users', user["id"], key) + for key in extra_fields: + uci.set('users', user["id"], key, extra_fields[key]) + uci.save("users") + return user["id"] + +def delete_remote_user(uci, name, database): + ''' + Delete an existing remote user + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Database identifier + + Returns: + - True if successful + ''' + user = get_user_by_name(uci, name, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', name) + if get_database_type(uci, database) != "ldap": + raise utils.ValidationError('database', 'db_not_ldap', database) + uci.delete('users', user["id"]) + uci.save("users") + return True + +def shadow_password(password): + ''' + Generate a shadow password + + Arguments: + - password -- Clear text password + + Returns: + - A shadow password in crypt(3) format, as generate by mkpasswd. Format: $6$salt$hash + ''' + return hash.sha512_crypt.using(salt=secrets.token_hex(8), rounds=5000).hash(password) + +def check_password(password, shadow): + ''' + Check a shadow password + + Arguments: + - password -- Clear text password + - shadow -- Shadow password in crypt(3) format + + Returns: + - True if password matches, False otherwise + ''' + return hash.sha512_crypt.verify(password, shadow) + + +def check_old_password(username, password): + """ + Checks if the old password is correct, discriminates between root and other users. + + Arguments: + - username -- Username of the user to check + - password -- Password to check + + Returns: + - True if the password is correct, False otherwise + """ + data = { + "username": username, + "password": password, + "timeout": 1 + } + process = subprocess.run(["/bin/ubus", "call", "session", "login", json.dumps(data)], capture_output=True) + return process.returncode == 0 + + +def ldif2users(ldif_data, user_attr="uid", display_attr="cn"): + ''' + Parse an LDIF file and return a list of users + + Arguments: + - ldif_data -- LDIF data + - user_attr -- User attribute (default: uid) + - display_attr -- Display name attr (default: cn) + + Returns: + - A list of users + ''' + users = [] + user = None + ldif_data_bytes = ldif_data.encode('utf-8') + ldif_data_io = BytesIO(ldif_data_bytes) + + parser = LDIFParser(ldif_data_io) + for dn, record in parser.parse(): + if user_attr in record: + user = {} + user["name"] = record[user_attr][0].lower() + if display_attr in record: + user["description"] = record[display_attr][0] + else: + user["description"] = "" + users.append(user) + return users + +def list_remote_users(uri, user_dn, user_attr, user_display_attr, start_tls=False, tls_reqcert="never", bind_dn=None, bind_password=None, schema='ldap'): + ''' + Test LDAP connection + + Arguments: + - uri -- LDAP URI + - user_dn -- LDAP user DN + - user_attr -- LDAP user attribute + - user_display_attr -- LDAP user full name attribute + - start_tls -- Use TLS (default: False) + - tls_reqcert -- TLS certificate validation (default: never) + - bind_dn -- LDAP bind DN + - bind_password -- LDAP bind password + - schema -- LDAP schema, 'ad' or 'ldap' + + Returns: + - A list of users, each one containing: + - name: user name + - description: user description + ''' + env = os.environ.copy() + env['LDAPTLS_REQCERT'] = tls_reqcert + omatch = '' + try: + # -E option executes a paged search without user prompt + # -LLL option suppresses LDAP version and search result headers + cmd = ["ldapsearch", "-LLL", "-x", "-H", uri, "-E", "pr=1000/noprompt", "-b", user_dn] + if start_tls: + cmd.append("-ZZ") + if bind_dn and bind_password: + cmd.extend(["-D", bind_dn, "-w", bind_password]) + if schema == "ad": + omatch = "(objectClass=person)" + else: + omatch = "(objectClass=posixAccount)" + cmd.extend(["-S", user_attr]) # request sorting + cmd.extend([omatch, user_display_attr, user_attr, user_display_attr]) + p = subprocess.run(cmd, env=env, capture_output=True, text=True) + return ldif2users(p.stdout, user_attr, user_display_attr) + except subprocess.CalledProcessError as e: + return [] + +def set_admin(uci, username, database): + ''' + Set a user as admin by creating a login record in rpcd configuration database + + Arguments: + - uci -- EUci pointer + - username -- User name + - database -- Database identifier + + Returns: + - The user identifier inside the rpcd configuration database + ''' + user = get_user_by_name(uci, username, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', username) + logins = utils.get_all_by_type(uci, 'rpcd', 'login') + for l in logins: + if logins[l].get("username") == username: + raise utils.ValidationError('name', 'admin_user_already_exists', username) + id = utils.get_random_id() + uci.set('rpcd', id, 'login') + uci.set('rpcd', id, 'username', username) + uci.set('rpcd', id, 'password', user["password"]) + uci.set('rpcd', id, 'read', '*') + uci.set('rpcd', id, 'write', '*') + uci.save("rpcd") + return id + +def remove_admin(uci, username): + ''' + Remove a user from rpcd configuration database + + Arguments: + - uci -- EUci pointer + - username -- User name + + Returns: + - True if successful + ''' + logins = utils.get_all_by_type(uci, 'rpcd', 'login') + for l in logins: + if logins[l].get("username") == username: + uci.delete('rpcd', l) + uci.save("rpcd") + return True + raise utils.ValidationError('name', 'admin_user_not_found', username) + +def is_admin(uci, username): + ''' + Check if a user is admin + + Arguments: + - uci -- EUci pointer + - username -- User name + + Returns: + - True if user is admin, False otherwise + ''' + logins = utils.get_all_by_type(uci, 'rpcd', 'login') + if logins is None: + return False + for l in logins: + if logins[l].get("username") == username: + return True + return False + + +def used_by(uci, database_name): + """ + Checks if the database is used by VPN or other services + + Arguments: + - uci -- EUci pointer + - database_name -- Database identifier + + Returns: + - dict containing the service that the database is used by + """ + results = [] + try: + for instance in uci.get_all('openvpn'): + if uci.get('openvpn', instance, 'ns_user_db', default='') == database_name: + results.append('openvpn') + break + except UciExceptionNotFound: + pass + + try: + for instance in uci.get_all('network'): + # could filter by proto = 'wireguard' but the performance is not an issue + if uci.get('network', instance, 'ns_user_db', default='') == database_name: + results.append('wireguard') + break + except UciExceptionNotFound: + pass + + return results diff --git a/packages/python3-nethsec/src/nethsec/utils/__init__.py b/packages/python3-nethsec/src/nethsec/utils/__init__.py new file mode 100644 index 000000000..fc3a85801 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/utils/__init__.py @@ -0,0 +1,555 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +General utilities +''' + +import os +import base64 +import re +import json +import uuid +import hashlib +import subprocess + +def get_random_id(): + ''' + Return a random valid UCI id. + + Random ids: + - have a length of 11 characters + - are sanitized accordingly to UCI conventions (see 'sanitize' function) + - start with ns\_ prefix + + Arguments: + - name -- the name of the section + + Returns: + - a valid UCI identifier as string + ''' + + h = hashlib.new('sha1') + h.update(uuid.uuid4().bytes) + digest = h.hexdigest() + return get_id(digest, 11) + +def get_id(name, length = 100): + ''' + Return a valid UCI id based on the given string. + All auto-generated NethSecurity ids: + + - have a maximum length of 100 characters + - start with ns\_ prefix + - are sanitized accordingly to UCI conventions + + Arguments: + - name -- the name of the section + - length -- maximum id length, default is 100. Maximum length for firewall zones is 15. + + Returns: + - a valid UCI identifier as string + ''' + sname = f'ns_{sanitize(name)}' + return sname[0:length] + +def sanitize(name): + ''' + Replace illegal chars with _ char. + UCI identifiers and config file names may contain only the characters a-z, 0-9 and _ + + Arguments: + - name -- the name of the section + + Returns: + - a string with valid chachars for UCI + ''' + name = re.sub(r'[^\x00-\x7F]+','_', name) + name = re.sub('[^0-9a-zA-Z]', '_', name) + name = name.removesuffix('_') + return name + +def get_all_by_type(uci, config, utype): + ''' + Return all sections of the given utype from the given config + + Arguments: + - uci -- EUci pointer + - config -- Configuration database name + - utype -- Section type + + Returns: + - A dictionary of all matched sections, None in case of error + ''' + ret = dict() + try: + for section in uci.get(config): + if uci.get(config, section) == utype: + ret[section] = uci.get_all(config, section) + return ret + except: + return None + +def get_device_name(hwaddr): + ''' + Retrieve the physical device name given the MAC address + + Aarguments: + - hwaddr -- MAC address string + + Returns: + - The device name as a string if the network interface has been found, None otherwise. + ''' + try: + interfaces = json.loads(subprocess.run(["/sbin/ip", "--json", "address", "show"], check=True, capture_output=True).stdout) + for interface in interfaces: + if interface["address"] == hwaddr: + return interface["ifname"] + except: + return None + + return None + +def get_interface_from_mac(uci, hwaddr): + ''' + Retrieve the logical UCI interface name given the MAC address + + Arguments: + - uci -- EUci pointer + - hwaddr -- MAC address string + + Returns: + - The device name as a string if the interface has been found, None otherwise + ''' + device = get_device_name(hwaddr) + return get_interface_from_device(uci, device) + +def get_interface_from_device(uci, device): + ''' + Retrieve the logical UCI interface name given the device name + + Arguments: + - uci -- EUci pointer + - device -- Device name + + Returns: + - The device name as a string if the interface has been found, None otherwise + ''' + for section in uci.get("network"): + if uci.get("network", section) == "interface": + try: + proto = uci.get("network", section, 'proto') + except: + continue + if proto == 'bonding' and section == device: + return section + if proto == 'pppoe' and section == device.removeprefix('pppoe-'): + return section + try: + sdevice = uci.get("network", section, 'device') + except: + continue + if proto != 'bonding' and device == sdevice: + return section + + return None + +def get_all_by_option(uci, config, option, value, deep = True): + ''' + Return all sections with the given option value + + Arguments: + - uci -- EUci pointer + - config -- Configuration database name + - option -- Option name + - value -- Option value + - deep - If true, return a dict of all matched keys, otherwise return a list of section names + + Returns: + - A dictionary or a list of all matched sections + ''' + ret = dict() + for section in uci.get(config, list=True, default=[]): + if uci.get(config, section, option, default='') == value: + if deep: + ret[section] = uci.get_all(config, section) + else: + ret[section] = 1 + if deep: + return ret + else: + return list(ret.keys()) + + +def get_all_devices_by_zone(uci, zone, exclude_aliases=False): + ''' + Retrieve all devices associated to the given zone + + Arguments: + - uci -- EUci pointer + - zone -- Firewall zone name + + Returns: + - A list of device names + ''' + devices = [] + for section in uci.get("firewall"): + if uci.get("firewall", section, default='') == 'zone' and uci.get("firewall", section, "name", default='') == zone: + devices = devices + list(uci.get("firewall", section, "device", list=True, default=[])) + networks = uci.get("firewall", section, "network", list=True, default=[]) + for network in networks: + if uci.get('network', network, 'proto', default='') == 'wireguard': + devices.append(network) + continue + device = uci.get("network", network, "device", default="") + if exclude_aliases and device.startswith("@"): + continue + if device != "": + devices.append(device) + else: + name = uci.get("network", network, "name", default="") + if name != "": + devices.append(name) + + # remove duplicates + return list(set(devices)) + + +def get_all_wan_devices(uci, exclude_aliases=False): + """ + Retrieve all devices associated to the wan zone + + Arguments: + - uci -- EUci pointer + - exclude_aliases -- If true, exclude devices starting with @ + + Returns: + - A list of device names + """ + return get_all_devices_by_zone(uci, 'wan', exclude_aliases) + + +def get_all_lan_devices(uci): + ''' + Retrieve all devices associated to the lan zone, except for VPN ones + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of device names + ''' + # exclude tun and ipsec devices + return list(filter(lambda d: not d.startswith("ipsec") and not d.startswith("tun"), get_all_devices_by_zone(uci, 'lan'))) + +def get_unassigned_devices(uci): + ''' + Retrieve all unused/unassigned devices. + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of devices + ''' + unassigned = [] + try: + p = subprocess.run(["ip", "-j", "link"], check=True, capture_output=True, text=True) + devices = json.loads(p.stdout) + except: + return [] + + u_interfaces = get_all_by_type(uci, 'network', 'interface') + u_devices = get_all_by_type(uci, 'network', 'device') + + for ip_device in devices: + free = True + ifname = ip_device.get('ifname', '') + # exclude special devices + if not ifname or ifname == "lo" or ifname.startswith("ifb-"): + continue + # exclude tun devices and ppp devices + if ip_device.get("link_type", "ether") == "none" or ip_device.get("link_type", "ether") == "ppp": + continue + # skip bridged interfaces + if not ip_device.get("master") is None: + continue + # search among UCI devices + for d in u_devices: + # ports are present on bridge devices + try: + ports = uci.get_all('network', d, 'ports') + except: + ports = [] + if uci.get('network', d, 'name', default="") == ifname or ifname in ports: + free = False + # search among UCI interfaces + for i in u_interfaces: + # slaves are present on bond devices + slaves = [] + if uci.get('network', i, 'proto', default='') == 'bonding': + slaves = list(uci.get_all('network', i, 'slaves')) + # for bonds, the section name is the nmame of device with 'bond-' prefix + slaves.append(f'bond-{i}') + device = uci.get('network', i, 'device', default="") + if device == ifname or ifname in slaves: + free = False + # search inside hotspot configuration + if uci.get('dedalo', 'config', 'disabled', default="1") == "0" and uci.get('dedalo', 'config', 'interface', default="") == ifname: + free = False + if free: + unassigned.append(ifname) + + # prepare list of devices used in bridges + used = {} + for d in u_devices: + try: + for p in uci.get_all('network', d, 'ports'): + used[p] = 1 + except: + pass + # prepare list of devices used in bonds + bonds = get_all_by_type(uci, 'network', 'interface') + for b in bonds: + try: + for s in uci.get_all('network', b, 'slaves'): + used[s] = 1 + except: + pass + + used = list(used.keys()) + + for d in u_devices: + free = True + d_name = uci.get('network', d, 'name', default=None) + # skip a device which is already up and visibile inside ip command + # check if the device is used inside and interface + if get_interface_from_device(uci, d_name): + continue + # check if the device is used inside a zone + for z in get_all_by_type(uci, 'firewall', 'zone'): + try: + z_devices = uci.get_all('firewall', z, 'device') + except: + z_devices = [] + try: + z_networks = uci.get_all('firewall', z, 'network') + except: + z_networks = [] + if d_name in z_devices or d_name in z_networks or d_name in used: + free = False + continue + if free: + unassigned.append(d_name) + return unassigned + + +def validation_errors(errors): + ''' + Generate a validation error for the APIs from an array. + + Arguments: + - errors -- An array of array errors + + Each array element is an array composed by 3 parameters: + - the name of the parameter + - the validation error reason + - the parameter original value that caused the error + + Returns: + - A validation error object + ''' + verrors = [] + for e in errors: + verrors.append({"parameter": e[0], "message": e[1].strip().replace(" ", "_").lower(), "value": e[2]}) + + return {"validation": {"errors": verrors}} + +def validation_error(parameter, message="", value=""): + ''' + Generate a validation error for the APIs. + + Arguments: + - parameter -- The name of the parameter + - message -- The validation error reason, default is empty + - value -- The parmeter original value that caused the error + + Returns: + - A validation error object + ''' + return validation_errors([[parameter, message, value]]) + +def generic_error(error): + ''' + Generate a generic error for the APIs. + + Arguments: + - message -- An error message + Returns: + - A validation error object + ''' + return {"error": error.strip().replace(" ", "_").lower()} + +def shadow_password(password): + ''' + Generates a shadow password hash using SHA-512 algorithm. + + Arguments: + - password (str) - the password to be hashed. + + Returns: + - the shadow password hash as strin + ''' + salt = base64.b64encode(os.urandom(12)) + phash = base64.b64encode(hashlib.pbkdf2_hmac('sha512', bytes(password, 'UTF-8'), salt, 200000)) + return f"$6${salt.decode('UTF-8')}${phash.decode('UTF-8')}" + +def check_password(password, shadow): + ''' + Check if the given password matches the given shadow password hash. + + Arguments: + - password (str) - the password to be checked. + - shadow (str) - the shadow password hash to be checked. + + Returns: + - True if the password matches the hash, False otherwise + ''' + (_, alg, salt, curhash) = shadow.split("$") + phash = base64.b64encode(hashlib.pbkdf2_hmac('sha512', bytes(password, 'UTF-8'), salt.encode("UTF-8"), 200000)) + return phash.decode("UTF-8") == curhash + +def get_user_by_username(uci, username): + ''' + Retrieve the user object id (UCI section) given the username + + Arguments: + - uci -- EUci pointer + - username -- Username + + Returns: + - The user object id (UCI section) if the user has been found, None otherwise + ''' + users = get_all_by_type(uci, 'objects', 'user') + for user in get_all_by_type(uci, "users", "user"): + if users[user].get("username", "") == username: + users[user]["id"] = user + return users[user] + return None + +def get_all_device_ips(): + ''' + Retrieve all device IPs + ''' + ret = {} + p = subprocess.run(["/sbin/ip", "-j", "address"], check=True, text=True, capture_output=True) + data = json.loads(p.stdout) + for interface in data: + ipaddr = [] + name = interface.get("ifname") + for addr in interface.get("addr_info", []): + if name not in ret: + ret[name] = [] + ret[name].append(addr.get('local', '')) + return ret + + +def get_all_wan_ips(uci): + ''' + List all WAN interfaces with their IP addresses + Exclude bond management IPs. + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of dictionaries with device and IP address + ''' + ret = [] + seen = set() # track seen devices to avoid duplicates + ips = get_all_device_ips() + wans = get_all_wan_devices(uci) + # list configured static wans + for wan in wans: + interface = get_interface_from_device(uci, wan) + if interface: + try: + ipaddrs = uci.get_all('network', interface, 'ipaddr') + except: + continue + if type(ipaddrs) == str: + ipaddrs = [(ipaddrs)] + for ipaddr in ipaddrs: + # skip bond management IPs + if ipaddr.startswith("127"): + continue + # strip mask to avoid duplicates + if ipaddr and '/' in ipaddr: + ipaddr = ipaddr.split('/')[0] + if ipaddr and ipaddr not in seen: + seen.add(ipaddr) + device = wan + if wan.startswith('@'): + device = uci.get('network', wan[1:], 'device', default='') + if device: + device = f'({device})' + ret.append({"device": f'{interface.replace("@","")} {device}', "ipaddr": ipaddr}) + # list on-line wans, search also for dynamic IPs + for device in ips.keys(): + if device in wans or device.startswith('pppoe'): + for ip in ips[device]: + # skip bond management IPs + if ip.startswith("127"): + continue + if ip not in seen: + seen.add(ip) + if device.startswith('pppoe-'): + interface = device.removeprefix('pppoe-') + else: + interface = get_interface_from_device(uci, device) + if interface: + ret.append({"device": f'{interface.replace("@","")} ({device})', "ipaddr": ip}) + + return sorted(ret, key=lambda k: k['ipaddr']) + + +def get_public_ip_addresses(ip_address=""): + """ + Return the public addresses associated to a private IP address. + + Arguments: + - ip_address -- the private IP address of a network interface. If not specified, the default network interface is used. + + Returns: + - a list of public addresses. Usually, the list contains only one element. + """ + try: + bindOption = "" + if ip_address: + bindOption = f" -b {ip_address}" + + cmd = f"/usr/bin/dig{bindOption} +short +time=1 myip.opendns.com @resolver1.opendns.com".split(" ") + output = subprocess.check_output(cmd, timeout=5) + public_ip_addresses = output.decode().strip().split('\n') + + if public_ip_addresses[0] == "": + # cannot retrieve public IP address, returning input IP address as fallback + if ip_address: + return [ip_address] + else: + return [] + else: + return public_ip_addresses + except Exception as e: + return [] + + +class ValidationError(ValueError): + def __init__(self, parameter, message="", value=""): + self.parameter = parameter + self.message = message + self.value = value diff --git a/packages/python3-nethsec/test.sh b/packages/python3-nethsec/test.sh new file mode 100755 index 000000000..3f8993ec2 --- /dev/null +++ b/packages/python3-nethsec/test.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# +# The container assumes the source code is mounted inside /app +# +IMAGE=ghcr.io/nethserver/nethsecurity/python3-nethsec-test +IMAGETAG=${IMAGETAG:-latest} +podman run --rm --tty --volume .:/app:Z "${IMAGE}:${IMAGETAG}" diff --git a/packages/python3-nethsec/tests/test_conntrack.py b/packages/python3-nethsec/tests/test_conntrack.py new file mode 100644 index 000000000..c1a5cf64f --- /dev/null +++ b/packages/python3-nethsec/tests/test_conntrack.py @@ -0,0 +1,455 @@ +from subprocess import CalledProcessError +from xml.etree import ElementTree + +import pytest +from pytest_mock import MockFixture + +from nethsec import conntrack + +conntrack_response = """ + + + + + 192.168.122.234 + 31.14.133.122 + + + 41692 + 123 + + + 1 + 76 + + + + + 31.14.133.122 + 192.168.122.234 + + + 123 + 41692 + + + 1 + 76 + + + + 47 + 16128 + 1 + 1905826093 + + + + + + 192.168.122.1 + 192.168.122.155 + + + + 2 + 168 + + + + + 192.168.122.155 + 192.168.122.1 + + + + 2 + 168 + + + + 29 + 16128 + 1 + 2860343346 + + + + + + 192.168.122.234 + 212.6.50.243 + + + 45538 + 123 + + + 1 + 76 + + + + + 212.6.50.243 + 192.168.122.234 + + + 123 + 45538 + + + 1 + 76 + + + + 48 + 16128 + 1 + 3555926027 + + + + + + 192.168.122.1 + 192.168.122.155 + + + 36578 + 22 + + + 73 + 7057 + + + + + 192.168.122.155 + 192.168.122.1 + + + 22 + 36578 + + + 54 + 10046 + + + + ESTABLISHED + 7440 + 16128 + 1 + 1285239412 + + + + + + + 192.168.122.234 + 93.49.6.247 + + + 50183 + 123 + + + 1 + 76 + + + + + 93.49.6.247 + 192.168.122.234 + + + 123 + 50183 + + + 1 + 76 + + + + 1 + 16128 + 1 + 3307146984 + + + + + + 192.168.122.1 + 192.168.122.255 + + + 57621 + 57621 + + + 1 + 72 + + + + + 192.168.122.255 + 192.168.122.1 + + + 57621 + 57621 + + + 0 + 0 + + + + 41 + 16128 + 1 + 3294787936 + + + + + + + 192.168.122.234 + 212.45.144.206 + + + 41960 + 123 + + + 1 + 76 + + + + + 212.45.144.206 + 192.168.122.234 + + + 123 + 41960 + + + 1 + 76 + + + + 48 + 16128 + 1 + 1915971940 + + + + + + 10.0.0.1 + 10.0.0.2 + + + 12345 + 80 + + + 5 + 500 + + + + + 10.0.0.2 + 10.0.0.1 + + + 80 + 12345 + + + 3 + 300 + + + + ESTABLISHED + 120 + 0 + 1 + 1111111111 + + + + + + + + + + 10.0.0.3 + 10.0.0.4 + + + 5353 + 53 + + + 1 + 60 + + + + + 10.0.0.4 + 10.0.0.3 + + + 53 + 5353 + + + 1 + 60 + + + + 30 + 0 + 1 + 2222222222 + + + + + + +""" + + +def test_connection_info(): + # get the first flow + flow = ElementTree.fromstring(conntrack_response).findall('flow')[0] + result = conntrack.__parse_connection_info(flow) + original = flow[0] + reply = flow[1] + assert result['source'] == '192.168.122.234' + assert result['destination'] == '31.14.133.122' + assert result['protocol'] == 'udp' + assert result['source_port'] == '41692' + assert result['destination_port'] == '123' + assert result['source_stats']['packets'] == 1 + assert result['source_stats']['bytes'] == 76 + assert result['destination_stats']['packets'] == 1 + assert result['destination_stats']['bytes'] == 76 + assert result['timeout'] == '47' + assert result['id'] == '1905826093' + flow = ElementTree.fromstring(conntrack_response).findall('flow')[1] + result = conntrack.__parse_connection_info(flow) + assert 'source_port' not in result + assert 'destination_port' not in result + + +def test_fetch_connection_list(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = conntrack_response + mocker.patch('subprocess.run', return_value=process_result) + result = conntrack.list_connections() + assert len(result) == 9 + + +def test_list_connections_label_filter(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = conntrack_response + subprocess_mock = mocker.patch('subprocess.run', return_value=process_result) + + # single label – '-l' flag appended with the label value + conntrack.list_connections(labels=['web']) + subprocess_mock.assert_called_with( + ['conntrack', '-L', '-o', 'labels,xml', '-l', 'web'], + capture_output=True, text=True + ) + + # multiple labels joined by comma + conntrack.list_connections(labels=['web', 'trusted']) + subprocess_mock.assert_called_with( + ['conntrack', '-L', '-o', 'labels,xml', '-l', 'web,trusted'], + capture_output=True, text=True + ) + + # no labels – no '-l' flag + conntrack.list_connections() + subprocess_mock.assert_called_with( + ['conntrack', '-L', '-o', 'labels,xml'], + capture_output=True, text=True + ) + + # empty list – no '-l' flag + conntrack.list_connections(labels=[]) + subprocess_mock.assert_called_with( + ['conntrack', '-L', '-o', 'labels,xml'], + capture_output=True, text=True + ) + + +def test_drop_connection(mocker: MockFixture): + connections_list = [ + { + 'protocol': 'udp', + 'source': '10.20.30.40', + 'destination': '1.1.1.1', + 'source_port': '123', + 'destination_port': '456', + 'id': '1234' + }, + { + 'protocol': 'icmp', + 'source': '1.1.1.1', + 'destination': '127.0.0.1', + 'id': '5678' + } + ] + mocker.patch('nethsec.conntrack.list_connections', return_value=connections_list) + command_line = mocker.patch('subprocess.run') + conntrack.drop_connection('1234') + command_line.assert_called_with([ + 'conntrack', '-D', '-p', 'udp', '-s', '10.20.30.40', '-d', '1.1.1.1', '--sport', '123', '--dport', '456' + ], check=True, capture_output=True) + conntrack.drop_connection('5678') + command_line.assert_called_with([ + 'conntrack', '-D', '-p', 'icmp', '-s', '1.1.1.1', '-d', '127.0.0.1' + ], check=True, capture_output=True) + with pytest.raises(ValueError) as e: + conntrack.drop_connection('9999') + + assert e.value.args[0] == "Connection with id 9999 not found." + with pytest.raises(RuntimeError) as e: + mocker.patch('subprocess.run', side_effect=CalledProcessError(1, 'conntrack')) + conntrack.drop_connection('1234') + + assert e.value.args[0].startswith("Error running command:") + + +def test_drop_all_connections(mocker: MockFixture): + command_line = mocker.patch('subprocess.run') + conntrack.drop_all_connections() + command_line.assert_called_with(['conntrack', '-F'], check=True, capture_output=True) + with pytest.raises(RuntimeError) as e: + mocker.patch('subprocess.run', side_effect=CalledProcessError(1, 'conntrack')) + conntrack.drop_all_connections() + + assert e.value.args[0].startswith("Error running command:") \ No newline at end of file diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py new file mode 100644 index 000000000..2e049dba2 --- /dev/null +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -0,0 +1,1067 @@ +import pathlib + +import pytest +from euci import EUci +from pytest_mock import MockFixture + +from nethsec import dpi +from nethsec.utils import ValidationError + +applications_file = """ +# Comment to avoid + +# another comment to avoid + +app:133:netify.netflix +app:10119:netify.linkedin +app:10552:netify.tesla +app:10195:netify.avira +app:10194:netify.sophos +app:10244:netify.bbc +app:10362:netify.hulu +app:10118:netify.lets-encrypt +app:199:netify.snapchat +dom:133:dualstack.apiproxy-device-prod-nlb-1-4d12762d4ba53e45.elb.eu-west-1.amazonaws.com +net:10852:208.118.237.0/24 +net:124:208.117.254.0/24 +nsd:-1:39:YXBwID09ICduZXRpZnkuc2lnbmFsJyAmJiBwcm90b2NvbF9pZCA9PSA3OCAmJiAob3RoZXJfcG9ydCA9PSA0NDMgfHwgbG9jYWxfcG9ydCA9PSA0NDMpOw== +nsd:-1:67:YXBwID09ICduZXRpZnkuZ29vZ2xlLWNoYXQnICYmIHByb3RvY29sX2lkICE9IDY3ICYmIChvdGhlcl9wb3J0ID09IDUyMjggfHwgbG9jYWxfcG9ydCA9PSA1MjI4KTs= +""" + +applications = { + 133: 'netify.netflix', + 10119: 'netify.linkedin', + 10552: 'netify.tesla', + 10195: 'netify.avira', + 10194: 'netify.sophos', + 10244: 'netify.bbc', + 10362: 'netify.hulu', + 10118: 'netify.lets-encrypt', + 199: 'netify.snapchat' +} + +protocols = { + 116: 'Warcraft3', + 117: 'LotusNotes', + 121: 'Dropbox', + 127: 'RPC', + 128: 'NetFlow', + 129: 'SFlow', + 130: 'HTTP/Connect' +} + +categories_file = """ +{ + "application_index": [ + [ + 3, + [ + 10362, + 10194, + 10552 + ] + ], + [ + 33, + [ + 133, + 10119, + 10195, + 10244 + ] + ], + [ + 20, + [ + 199 + ] + ] + ], + "application_tag_index": { + "first-category": 3, + "unknown": 33, + "last": 20 + }, + "protocol_index": [ + [ + 1, + [ + 121, + 127, + 128 + ] + ], + [ + 2, + [ + 116 + ] + ], + [ + 4, + [ + 129, + 130 + ] + ] + ], + "protocol_tag_index": { + "base": 1, + "games": 2, + "low": 4 + } +} +""" + +categories_file_old = """ +{ + "last_update": 1696950440, + "application_index": { + "first-category": [ + 10362, + 10194, + 10552 + ], + "unknown": [ + 133, + 10119, + 10195, + 10244 + ], + "last": [ + 199 + ] + }, + "protocol_index": { + "base": [ + 121, + 127, + 128 + ], + "games": [ + 116 + ], + "low": [ + 129, + 130 + ] + } +} +""" + +application_categories = { + 10362: { + 'name': 'first-category' + }, + 10194: { + 'name': 'first-category' + }, + 10552: { + 'name': 'first-category' + }, + 133: { + 'name': 'unknown' + }, + 10119: { + 'name': 'unknown' + }, + 10195: { + 'name': 'unknown' + }, + 10244: { + 'name': 'unknown' + }, + 199: { + 'name': 'last' + } +} + +protocol_categories = { + 121: { + 'name': 'base' + }, + 127: { + 'name': 'base' + }, + 128: { + 'name': 'base' + }, + 116: { + 'name': 'games' + }, + 129: { + 'name': 'low' + }, + 130: { + 'name': 'low' + } +} + +protocol_output = """ + 116: Warcraft3 + 117: LotusNotes + 121: Dropbox + 127: RPC + 128: NetFlow + 129: SFlow + 130: HTTP/Connect +""" + +dpi_minimal_db = """ +config main 'config' + option log_blocked '0' + option firewall_exemption '0' + option enabled '0' +""" + +dpi_db = """ +config main 'config' + option log_blocked '0' + option firewall_exemption '0' + option enabled '0' + list popular_filters 'netify.netflix' + list popular_filters 'netify.hulu' + list popular_filters 'netify.whatsapp' + list popular_filters 'netify.facebook' + list popular_filters 'netify.sophos' + list popular_filters 'HTTP/Connect' + list popular_filters 'Dropbox' + +config rule rule0 + option action 'block' + list application 'netify.linkedin' + list application 'netify.snapchat' + list protocol 'HTTP/Connect' + list source '192.168.100.1' + list source '192.168.100.2' + list source 'user:giacomo' + list source 'group:g1' + list category 'games' + option device 'eth0' + option enabled 1 + +config rule rule1 + option action 'block' + list application 'netify.tesla' + option device 'eth4' + option enabled 0 + list exemption '192.168.100.3' + +config rule rule2 + option action 'block' + option criteria 'local_ip == 192.168.100.22 && application == "netify.facebook";' + option enabled 1 + +config rule rule3 + option action 'video' + list protocol 'HTTP/Connect' + option device 'eth1' + option enabled 1 + +config exemption exemp1 + option criteria '192.168.1.1' + option description 'my host' + option enabled 1 +""" + +netifyd_config = """ +config netifyd + option enabled '1' + option autoconfig '0' + list internal_if 'br-lan' + list internal_if 'eth0' + list internal_if 'eth4' + list internal_if 'tunrw1' + list external_if 'eth2' + list external_if 'eth1' +""" + +network_config = """ +config interface 'RED_1' + option proto 'dhcp' + option device 'eth1' + option metric '1' + +config interface 'RED_2' + option proto 'dhcp' + option device 'eth2' + option metric '2' + +config interface 'RED_3' + option proto 'dhcp' + option device 'eth3' + option metric '3' + +config interface 'GREEN_1' + option proto 'static' + option device 'eth0' + option ipaddr '192.168.200.2' + option netmask '255.255.255.0' + option gateway '192.168.200.1' + +config interface 'GREEN_2' + option proto 'static' + option device 'eth4' + option ipaddr '10.0.3.1' + option netmask '255.255.255.0' + option gateway '10.0.3.0' + +config interface 'GREEN_3' + option proto 'static' + option device 'br_lan' + option ipaddr '10.1.1.1' + option netmask '255.255.255.0' + option gateway '10.1.1.0' +""" + +firewall_config = """ +config zone 'ns_lan' + option name 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list network 'GREEN_1' + list network 'GREEN_2' + +config zone 'ns_wan' + option name 'wan' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + list network 'RED_1' + list network 'RED_2' + list network 'RED_3' + +config zone 'ns_guests' + option name 'guests' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + list network 'GREEN_1' + +config zone 'ns_empty' + option name 'guests' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' +""" + + +@pytest.fixture +def e_uci(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + with conf_dir.joinpath('dpi').open('w') as fp: + fp.write(dpi_minimal_db) + with conf_dir.joinpath('network').open('w') as fp: + fp.write(network_config) + with conf_dir.joinpath('firewall').open('w') as fp: + fp.write(firewall_config) + return EUci(confdir=conf_dir.as_posix(), savedir=save_dir.as_posix()) + + +@pytest.fixture +def e_uci_with_data(e_uci: EUci): + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(dpi_db) + with pathlib.Path(e_uci.confdir()).joinpath('netifyd').open('w') as fp: + fp.write(netifyd_config) + return e_uci + + +@pytest.fixture +def mock_load(mocker): + mocker.patch('nethsec.dpi.__load_applications', return_value=applications) + mocker.patch('nethsec.dpi.__load_application_categories', return_value=application_categories) + mocker.patch('nethsec.dpi.load_protocols', return_value=protocols) + mocker.patch('nethsec.dpi.__load_protocol_categories', return_value=protocol_categories) + + +def test_load_applications(mocker: MockFixture): + mocker.patch('builtins.open', mocker.mock_open(read_data=applications_file)) + assert dpi.__load_applications() == applications + + +@pytest.mark.parametrize('data', [categories_file, categories_file_old]) +def test_load_application_categories(mocker: MockFixture, data: str): + mocker.patch('builtins.open', mocker.mock_open(read_data=data)) + assert dpi.__load_application_categories() == application_categories + + +@pytest.mark.parametrize('data', [categories_file, categories_file_old]) +def test_load_protocol_categories(mocker: MockFixture, data: str): + mocker.patch('builtins.open', mocker.mock_open(read_data=data)) + assert dpi.__load_protocol_categories() == protocol_categories + + +def test_load_protocols(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = bytes(protocol_output, 'utf-8') + mocker.patch('subprocess.run', return_value=process_result) + assert dpi.load_protocols() == protocols + + +@pytest.mark.parametrize('search', [None, '']) +def test_index_applications(mock_load, search): + assert dpi.list_applications(search) == { + 'data': [ + { + 'id': 133, + 'name': 'netify.netflix', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10119, + 'name': 'netify.linkedin', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10195, + 'name': 'netify.avira', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10194, + 'name': 'netify.sophos', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10244, + 'name': 'netify.bbc', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10362, + 'name': 'netify.hulu', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10118, + 'name': 'netify.lets-encrypt', + 'type': 'application' + }, + { + 'id': 199, + 'name': 'netify.snapchat', + 'type': 'application', + 'category': { + 'name': 'last' + } + }, + { + 'id': 116, + 'name': 'Warcraft3', + 'type': 'protocol', + 'category': { + 'name': 'games' + } + }, + { + 'id': 117, + 'name': 'LotusNotes', + 'type': 'protocol' + }, + { + 'id': 121, + 'name': 'Dropbox', + 'type': 'protocol', + 'category': { + 'name': 'base' + } + }, + { + 'id': 127, + 'name': 'RPC', + 'type': 'protocol', + 'category': { + 'name': 'base' + } + }, + { + 'id': 128, + 'name': 'NetFlow', + 'type': 'protocol', + 'category': { + 'name': 'base' + } + }, + { + 'id': 129, + 'name': 'SFlow', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ], + 'meta': { + 'last_page': 1, + 'total': 16 + } + } + + +def test_index_applications_search(mock_load): + assert dpi.list_applications(search='l') == { + 'data': [ + { + 'id': 133, + 'name': 'netify.netflix', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10119, + 'name': 'netify.linkedin', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10362, + 'name': 'netify.hulu', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10118, + 'name': 'netify.lets-encrypt', + 'type': 'application' + }, + { + 'id': 199, + 'name': 'netify.snapchat', + 'type': 'application', + 'category': { + 'name': 'last' + } + }, + { + 'id': 117, + 'name': 'LotusNotes', + 'type': 'protocol' + }, + { + 'id': 128, + 'name': 'NetFlow', + 'type': 'protocol', + 'category': { + 'name': 'base' + } + }, + { + 'id': 129, + 'name': 'SFlow', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ], + 'meta': { + 'last_page': 1, + 'total': 10 + } + } + + +def test_index_applications_paginate(mock_load): + assert dpi.list_applications(limit=2, page=2) == { + 'data': [ + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10195, + 'name': 'netify.avira', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + } + ], + 'meta': { + 'last_page': 8, + 'total': 16 + } + } + assert dpi.list_applications(limit=2, page=3) == { + 'data': [ + { + 'id': 10194, + 'name': 'netify.sophos', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10244, + 'name': 'netify.bbc', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + } + ], + 'meta': { + 'last_page': 8, + 'total': 16 + } + } + + +def test_list_empty_rules(e_uci, mock_load): + assert dpi.list_rules(e_uci) == [] + + +def test_list_rules(e_uci_with_data, mock_load): + assert dpi.list_rules(e_uci_with_data) == [ + { + 'config-name': 'rule0', + 'enabled': True, + 'interface': 'GREEN_1', + 'device': 'eth0', + 'action': 'block', + 'criteria': [ + { + 'id': 10119, + 'name': 'netify.linkedin', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 199, + 'name': 'netify.snapchat', + 'type': 'application', + 'category': { + 'name': 'last' + } + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + }, + { + 'config-name': 'rule1', + 'enabled': False, + 'interface': 'GREEN_2', + 'device': 'eth4', + 'action': 'block', + 'criteria': [ + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + } + ] + }, + { + 'config-name': 'rule3', + 'enabled': True, + 'interface': 'RED_1', + 'device': 'eth1', + 'action': 'video', + 'criteria': [ + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + } + ] + + +def test_store_rule(e_uci, mock_load): + rule_created = dpi.add_rule(e_uci, True, 'eth0', 'best_effort', + ['netify.linkedin', 'netify.avira', 'netify.netflix'], ['LotusNotes', 'SFlow']) + assert dpi.list_rules(e_uci) == [ + { + 'config-name': rule_created, + 'enabled': True, + 'interface': 'GREEN_1', + 'device': 'eth0', + 'action': 'best_effort', + 'criteria': [ + { + 'id': 10119, + 'name': 'netify.linkedin', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10195, + 'name': 'netify.avira', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 133, + 'name': 'netify.netflix', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 117, + 'name': 'LotusNotes', + 'type': 'protocol', + }, + { + 'id': 129, + 'name': 'SFlow', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + } + ] + assert(e_uci.get("dpi", "config", "enabled") == "1") + + +def test_delete_rule(e_uci_with_data, mock_load): + dpi.delete_rule(e_uci_with_data, 'rule1') + dpi.delete_rule(e_uci_with_data, 'rule0') + assert dpi.list_rules(e_uci_with_data) == [ + { + 'config-name': 'rule3', + 'enabled': True, + 'interface': 'RED_1', + 'device': 'eth1', + 'action': 'video', + 'criteria': [ + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + } + ] + dpi.delete_rule(e_uci_with_data, 'rule2') + dpi.delete_rule(e_uci_with_data, 'rule3') + assert(e_uci_with_data.get("dpi", "config", "enabled") == "0") + + +def test_edit_rule(e_uci_with_data, mock_load): + dpi.edit_rule(e_uci_with_data, 'rule0', False, 'eth0', 'voice', [], + ['HTTP/Connect', 'LotusNotes']) + assert dpi.list_rules(e_uci_with_data) == [ + { + 'config-name': 'rule0', + 'enabled': False, + 'interface': 'GREEN_1', + 'device': 'eth0', + 'action': 'voice', + 'criteria': [ + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + }, + { + 'id': 117, + 'name': 'LotusNotes', + 'type': 'protocol', + } + ] + }, + { + 'config-name': 'rule1', + 'enabled': False, + 'interface': 'GREEN_2', + 'device': 'eth4', + 'action': 'block', + 'criteria': [ + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + } + ] + }, + { + 'config-name': 'rule3', + 'enabled': True, + 'interface': 'RED_1', + 'device': 'eth1', + 'action': 'video', + 'criteria': [ + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + } + ] + + +def test_edit_rule_with_missing_rule(e_uci): + with pytest.raises(ValidationError) as err: + dpi.edit_rule(e_uci, 'rule0', False, 'eth0', 'block', [], []) + + assert err.value.args[0] == 'config-name' + assert err.value.args[1] == 'invalid' + assert err.value.args[2] == 'rule0' + + +def test_list_interfaces(e_uci_with_data): + assert dpi.list_devices(e_uci_with_data).index({ + 'interface': 'GREEN_1', + 'device': 'eth0' + }) != -1 + assert dpi.list_devices(e_uci_with_data).index({ + 'interface': 'GREEN_2', + 'device': 'eth4' + }) != -1 + with pytest.raises(ValueError): + assert dpi.list_devices(e_uci_with_data).index({ + 'interface': 'RED_1', + 'device': 'eth1' + }) + + +def test_list_popular(e_uci_with_data, mock_load): + assert dpi.list_popular(e_uci_with_data) == { + 'data': [ + { + 'id': 133, + 'name': 'netify.netflix', + 'type': 'application', + 'category': { + 'name': 'unknown' + }, + 'missing': False + }, + { + 'id': 10362, + 'name': 'netify.hulu', + 'type': 'application', + 'category': { + 'name': 'first-category' + }, + 'missing': False + }, + { + 'name': 'netify.whatsapp', + 'missing': True + }, + { + 'name': 'netify.facebook', + 'missing': True + }, + { + 'id': 10194, + 'name': 'netify.sophos', + 'type': 'application', + 'category': { + 'name': 'first-category' + }, + 'missing': False + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + }, + 'missing': False + }, + { + 'id': 121, + 'name': 'Dropbox', + 'type': 'protocol', + 'category': { + 'name': 'base' + }, + 'missing': False + } + ], + 'meta': { + 'last_page': 1, + 'total': 7 + } + } + + +def test_list_popular_with_limits(e_uci_with_data, mock_load): + assert dpi.list_popular(e_uci_with_data, limit=2, page=2) == { + 'data': [ + { + 'name': 'netify.whatsapp', + 'missing': True + }, + { + 'name': 'netify.facebook', + 'missing': True + } + ], + 'meta': { + 'last_page': 4, + 'total': 7 + } + } + assert dpi.list_popular(e_uci_with_data, limit=2, page=3) == { + 'data': [ + { + 'id': 10194, + 'name': 'netify.sophos', + 'type': 'application', + 'category': { + 'name': 'first-category' + }, + 'missing': False + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + }, + 'missing': False + } + ], + 'meta': { + 'last_page': 4, + 'total': 7 + } + } + +def test_list_exemptions(e_uci_with_data): + assert dpi.list_exemptions(e_uci_with_data) == [ + { + 'config-name': 'exemp1', + 'enabled': True, + 'criteria': '192.168.1.1', + 'description': 'my host', + } + ] + +def test_add_exemption(e_uci): + ex_created = dpi.add_exemption(e_uci, "192.168.2.2", 'my host2', True) + assert dpi.list_exemptions(e_uci) == [ + { + 'config-name': ex_created, + 'enabled': True, + 'criteria': "192.168.2.2", + 'description': 'my host2', + } + ] + with pytest.raises(ValidationError) as err: + dpi.add_exemption(e_uci, "192.168.2.2", 'duplicated', True) + +def test_edit_exemption(e_uci_with_data): + dpi.edit_exemption(e_uci_with_data, 'exemp1', '192.168.1.3', 'my host 3', False) + assert dpi.list_exemptions(e_uci_with_data) == [ + { + 'config-name': 'exemp1', + 'enabled': False, + 'criteria': "192.168.1.3", + 'description': 'my host 3', + } + ] + +def test_delete_exemption(e_uci): + dpi.delete_exemption(e_uci, 'exemp1') + assert dpi.list_exemptions(e_uci) == [] diff --git a/packages/python3-nethsec/tests/test_firewall.py b/packages/python3-nethsec/tests/test_firewall.py new file mode 100644 index 000000000..0920bcb31 --- /dev/null +++ b/packages/python3-nethsec/tests/test_firewall.py @@ -0,0 +1,1262 @@ +from nethsec import utils +from nethsec.utils import ValidationError +import pytest +from euci import EUci, UciExceptionNotFound + +from nethsec import firewall, objects +from pytest_mock import MockFixture +from unittest.mock import MagicMock, patch +import pathlib +firewall_db = """ +config zone lan1 + option name 'lan' + list network 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list device 'eth0' + +config zone grey + option name 'grey' + list network 'grey1' + option input 'DROP' + option output 'ACCEPT' + option forward 'DROP' + list device 'eth1' + +config zone orange + option name 'orange' + list network 'orange1' + option input 'DROP' + option output 'DROP' + option forward 'DROP' + +config zone wan1f + option name 'wan' + list network 'wan' + list network 'wan6' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + +config zone with_log + option name 'with_log' + option input 'DROP' + option output 'DROP' + option forward 'DROP' + option log '1' + +config forwarding fw1 + option src 'lan' + option dest 'wan' + +config rule 'v6rule' + option name 'Allow-DHCPv6' + option src 'wan' + option proto 'udp' + option dest_port '546' + option family 'ipv6' + option target 'ACCEPT' + +config rule 'manrule' + option name 'Not-Automated' + option src 'wan' + option proto 'tcp' + option dest_port '1234' + option family 'ipv4' + option target 'ACCEPT' + +config rule 'f1' + option name 'r1' + option dest 'wan' + option dest_port '22' + option target 'ACCEPT' + option src 'lan' + list src_ip '192.168.100.1' + list src_ip '192.168.100.238' + list dest_ip '192.168.122.1' + list dest_ip '192.168.122.49' + option log '1' + +config rule 'o1' + option name 'output1' + list dest_ip '192.168.100.1' + option target 'ACCEPT' + option dest 'wan' + +config rule 'i1' + option name 'Allow-OpenVPNRW1' + option src 'wan' + option dest_port '1194' + option proto 'udp' + option target 'ACCEPT' + list ns_tag 'automated' + option ns_link 'openvpn/ns_roadwarrior1' + +config nat + option name 'source_NAT1_vpm' + option src '*' + option src_ip '192.168.55.0/24' + option dest_ip '10.20.30.0/24' + option target 'SNAT' + option snat_ip '10.44.44.1' + list proto 'all' + +config nat + option name 'masquerade' + list proto 'all' + option src 'lan' + option src_ip '192.168.1.0/24' + option dest_ip '10.88.88.0/24' + option target 'MASQUERADE' + +config nat + option name 'cdn_via_router' + list proto 'all' + option src 'lan' + option src_ip '192.168.1.0/24' + option dest_ip '192.168.50.0/24' + option target 'ACCEPT' + +config nat + option name 'SNAT_NSEC7_style' + list proto 'all' + option src 'wan' + option src_ip '192.168.1.44' + option target 'SNAT' + option snat_ip '10.20.30.5' + +config redirect 'redirect3' + option ns_src '' + option ipset 'redirect3_ipset' + +config ipset 'redirect3_ipset' + option name 'redirect1_ipset' + option match 'src_net' + option enabled '1' + list entry '6.7.8.9' + +config redirect 'redirect4' + option ns_src '' + +config rule 'r1' + option name 'r1' + option ns_dst 'dhcp/ns_8dcab636' + +config rule 'r2' + option name 'r2' + option ns_dst '' + +config rule 'r3' + option name 'r3' + option ns_src '' + +config rule 'r4' + option ns_dst '' + option ns_src '' + +config redirect 'ns_59eea167' + option src 'wan' + option target 'DNAT' + option dest_ip '192.168.1.22' + option dest_port '4466' + option enabled '1' + option log '1' + option name 'pf1' + list proto 'tcp' + list proto 'udp' + option reflection '0' + option src_dport '4455' +""" + +network_db = """ +config interface lan + option device 'vnet3' + +config interface lan2 + option device 'vnet4' + +config interface grey1 + option device 'vnet5' + +config interface orange1 + option device 'vnet6' + +config interface 'lan6' + option device 'br-lan' + option proto 'dhcpv6' + +config device 'vlan6' + option type '8021ad' + option ifname 'eth2' + option vid '1' + option name 'eth2.1' + option ipv6 '1' + +config interface 'wan6' + option device eth1 + option proto static + option ip6addr 2001:db80::2/64 + option ip6gw 2001:db80::1 + option ip6prefix 2001:db80:1::/48 + option dns 2001:db80::1 + +config interface 'wan6b' + option device eth1 + option proto vtiv6 + +config interface 'wan6c' + option device eth44 + option ipv6 1 + +config interface 'bond1' + option proto 'bonding' + option ipaddr '10.0.0.22' + option netmask '255.255.255.0' + list slaves 'eth3' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + option all_slaves_active '0' + option link_monitoring 'off' + +config interface 'lan' + option device 'br-lan' + option proto 'static' + option ipaddr '192.168.100.238' + option netmask '255.255.255.0' + +config interface 'al_lan' + option proto 'static' + option device '@lan' + list ipaddr '10.0.0.1/24' +""" + +templates_db = """ +# Service groups +config template_service_group 'ns_web_secure' + option name 'Secure web navigation' + list services '80/tcp/HTTP' + list services '443/tcp/HTTP Secure' + list services '53/udp/DNS' + +# Blue zone +config template_zone 'ns_blue' + option name 'blue' + option forward 'DROP' + option input 'DROP' + option output 'ACCEPT' + option ns_description 'Guest network with Internet access' + list forwardings 'ns_blue2wan' + list forwardings 'ns_blue2lan' + # requires network option + +config template_forwarding 'ns_blue2wan' + option src 'blue' + option dest 'wan' + +config template_forwarding 'ns_blue2lan' + option src 'blue' + option dest 'lan' + +# Default rule +config template_rule 'ns_test_rule' + option name 'Test-rule' + option src 'wan' + option dest 'blue' + option dest_port '__PORT__' + option proto '__PROTO__' + option target 'ACCEPT' + option enabled '1' + +# IPv6 rules + +config template_rule 'ip6_dhcp' + option name 'Allow-DHCPv6' + option src 'wan' + option proto 'udp' + option dest_port '546' + option family 'ipv6' + option target 'ACCEPT' + +config template_rule 'ip6_mld' + option name 'Allow-MLD' + option src 'wan' + option proto 'icmp' + option src_ip 'fe80::/10' + list icmp_type '130/0' + list icmp_type '131/0' + list icmp_type '132/0' + list icmp_type '143/0' + option family 'ipv6' + option target 'ACCEPT' + +config template_rule 'ip6_icmp' + option name 'Allow-ICMPv6-Input' + option src 'wan' + option proto 'icmp' + list icmp_type 'echo-request' + list icmp_type 'echo-reply' + list icmp_type 'destination-unreachable' + list icmp_type 'packet-too-big' + list icmp_type 'time-exceeded' + list icmp_type 'bad-header' + list icmp_type 'unknown-header-type' + list icmp_type 'router-solicitation' + list icmp_type 'neighbour-solicitation' + list icmp_type 'router-advertisement' + list icmp_type 'neighbour-advertisement' + option limit '1000/sec' + option family 'ipv6' + option target 'ACCEPT' + +config template_rule 'ip6_icmp_forward' + option name 'Allow-ICMPv6-Forward' + option src 'wan' + option dest '*' + option proto 'icmp' + list icmp_type 'echo-request' + list icmp_type 'echo-reply' + list icmp_type 'destination-unreachable' + list icmp_type 'packet-too-big' + list icmp_type 'time-exceeded' + list icmp_type 'bad-header' + list icmp_type 'unknown-header-type' + option limit '1000/sec' + option family 'ipv6' + option target 'ACCEPT' + +""" + +zone_testing_db = """ +config zone 'ns_lan' + option name 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list network 'GREEN_1' + list network 'al_lan' + +config zone 'ns_wan' + option name 'wan' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + option log '1' + list network 'wan6' + list network 'RED_2' + list network 'RED_3' + list network 'RED_1' + +config zone 'ns_guests' + option name 'guests' + option input 'DROP' + option forward 'DROP' + option output 'ACCEPT' + +config forwarding + option src 'lan' + option dest 'wan' + +config forwarding 'ns_guests2wan' + option src 'guests' + option dest 'wan' + +config forwarding 'ns_lan2guests' + option src 'lan' + option dest 'guests' +""" + +dhcp_db = """ +config domain + option name 'test.name.org' + option ip '192.168.100.1' + +config host + option name 'test2.giacomo.org' + option mac '5c:87:9c:fa:69:5b' + option ip '192.168.100.2' + +config domain + option name 'test3.test.org' + option ip 'ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543' + +config domain 'ns_dhcp1' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' +""" + +netmap_db = """ +config rule + option name 'source_nat1' + option dest '10.50.50.0/24' + list device_in 'eth0' + list device_in 'eth1' + list device_out 'tunrw' + option map_from '192.168.1.0/24' + option map_to '192.168.57.0/24' + +config rule + option name 'dest_nat1' + option src '10.50.50.0/24' + option map_from '192.168.1.0/24' + option map_to '192.168.57.0/24' +""" + +services_file = """ +ftp 21/tcp +ssh 22/tcp +ssh 22/udp +time 37/udp +www 80/tcp http +kerberos 88/tcp kerberos5 krb5 kerberos-sec +kerberos 88/udp kerberos5 krb5 kerberos-sec +""" + +lease_file = """ +1704890657 9c:3d:cf:ea:94:9b 192.168.1.70 * * +1704885018 98:ed:5c:8c:73:2a 192.168.1.228 test1 * +1704874398 ac:57:26:00:24:8c 192.168.1.219 test2 01:dc:57:26:00:25:8c +""" + +objects_db = """ +""" + +user_db = """ +config user 'ns_user1' + option name "john" + option database "main" + option label "John Doe" + option openvpn_ipaddr "10.10.10.22" +""" + +mwan3_db = """ +""" + +dpi_db = """ +""" + +# Setup fake ip command output +ip_json='[{"ifindex":9,"ifname":"vnet3","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"virbr2","operstate":"UNKNOWN","group":"default","txqlen":1000,"link_type":"ether","address":"fe:62:31:19:0b:29","broadcast":"ff:ff:ff:ff:ff:ff","addr_info":[{"family":"inet6","local":"fe80::fc62:31ff:fe19:b29","prefixlen":64,"scope":"link","valid_life_time":4294967295,"preferred_life_time":4294967295}]},{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":3,"ifname":"eth1","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":4,"ifname":"eth2","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:75:1c:c1","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":5,"ifname":"eth3","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":6,"ifname":"ifb-dns","flags":["BROADCAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":32,"link_type":"ether","address":"72:79:65:12:07:07","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":7,"ifname":"br-lan","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":9,"ifname":"bond-bond1","flags":["BROADCAST","MULTICAST","MASTER","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":8,"ifname":"tuntunsubnet","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":500,"link_type":"none"}, {"ifindex":69,"ifname":"pppoe-w1","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1492,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":3,"link_type":"ppp"},{"ifindex":20,"link":"eth1","ifname":"eth1.4","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"}]' +mock_ip_stdout = MagicMock() +mock_ip_stdout.configure_mock(**{"stdout": ip_json}) + +@pytest.fixture +def u(tmp_path: pathlib.Path) -> EUci: + # setup fake dbs + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + fp.write(zone_testing_db) + with tmp_path.joinpath('network').open('w') as fp: + fp.write(network_db) + with tmp_path.joinpath('templates').open('w') as fp: + fp.write(templates_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('netmap').open('w') as fp: + fp.write(netmap_db) + with tmp_path.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with tmp_path.joinpath('users').open('w') as fp: + fp.write(user_db) + with tmp_path.joinpath('mwan3').open('w') as fp: + fp.write(mwan3_db) + with tmp_path.joinpath('dpi').open('w') as fp: + fp.write(dpi_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_add_interface_to_zone(u): + z1 = firewall.add_interface_to_zone(u, "interface1", "lan") + assert z1 == 'lan1' + assert 'interface1' in u.get_all('firewall', 'lan1', 'network') + assert firewall.add_interface_to_zone(u, "interface1", "blue") == None + z1 = firewall.add_interface_to_zone(u, "interface2", "lan") + assert 'interface2' in u.get_all('firewall', 'lan1', 'network') + +def test_remove_interface_from_zone(u): + z1 = firewall.remove_interface_from_zone(u, 'interface1', "lan") + assert(not 'interface1' in u.get_all('firewall', 'lan1', 'network')) + +def test_add_device_to_zone(u): + z1 = firewall.add_device_to_zone(u, "vnet1", "lan") + assert z1 == 'lan1' + assert 'vnet1' in u.get_all('firewall', 'lan1', 'device') + assert firewall.add_device_to_zone(u, "vnet1", "blue") == None + +def test_add_device_to_lan(u): + assert firewall.add_device_to_lan(u, "vnet1") == 'lan1' + assert 'vnet1' in u.get_all('firewall', 'lan1', 'device') + +def test_add_device_to_wan(u): + assert firewall.add_device_to_wan(u, "vnet2") == 'wan1f' + assert 'vnet2' in u.get_all('firewall', 'wan1f', 'device') + +def test_remove_device_from_zone(u): + firewall.remove_device_from_zone(u, 'vnet1', "lan") + assert(not 'vnet2' in u.get_all('firewall', 'lan1', 'device')) + +def test_add_service(u): + rule = firewall.add_service(u, "my_service", "443", "tcp", "nginx/_lan") + assert rule is not None + assert rule == "ns_allow_my_service" + assert u.get('firewall', rule, 'dest_port') == '443' + assert u.get('firewall', rule, 'proto') == 'tcp' + assert u.get('firewall', rule, 'src') == 'wan' + assert u.get('firewall', rule, 'name') == 'Allow-my_service' + assert u.get('firewall', rule, 'ns_link') == "nginx/_lan" + assert u.get('firewall', rule, 'ns_tag') == "automated" + + assert firewall.add_service(u, "my-service2", "456", ["tcp", "udp"]) == "ns_allow_my_service2" + assert u.get_all('firewall', 'ns_allow_my_service2', 'proto') == ("tcp", "udp") + +def test_block_service(u): + firewall.add_service(u, "my-service", "443", "tcp") + assert firewall.remove_service(u, "my-service") == "ns_allow_my_service" + with pytest.raises(UciExceptionNotFound): + u.get('firewall', 'ns_allow_my_service') + +def test_disable_service(u): + firewall.add_service(u, "my-service", "443", "tcp") + assert firewall.disable_service(u, "my-service") == "ns_allow_my_service" + assert u.get("firewall", "ns_allow_my_service", "enabled") == "0" + assert firewall.disable_service(u, "non-existing") == None + +def test_enable_service(u): + firewall.add_service(u, "my-service", "1234", "tcp") + firewall.disable_service(u, "my-service") + assert firewall.enable_service(u, "my-service") == "ns_allow_my_service" + assert u.get("firewall", "ns_allow_my_service", "enabled") + assert firewall.enable_service(u, "non-existing") == None + +def test_add_vpn_interface(u): + assert firewall.add_vpn_interface(u, 'test!vpn', 'tuntest') == 'test_vpn' + assert u.get('network', 'test_vpn') == 'interface' + assert u.get('network', 'test_vpn', 'proto') == 'none' + assert u.get('network', 'test_vpn', 'device') == 'tuntest' + assert u.get('network', 'test_vpn', 'ns_tag') == 'automated' + i = firewall.add_vpn_interface(u, 'p2p', 'ppp10', 'torrent/server1') + assert u.get('network', 'p2p', 'ns_link') == 'torrent/server1' + +def test_add_trusted_zone(u): + assert firewall.add_trusted_zone(u, 'toolongnameforzone') == (None, None) + + (zone, forwardings) = firewall.add_trusted_zone(u, 'mytrusted') + assert zone is not None + assert len(forwardings) == 3 + assert u.get("firewall", zone) == "zone" + assert u.get("firewall", zone, 'name') == "mytrusted" + assert u.get("firewall", zone, 'input') == "ACCEPT" + assert u.get("firewall", zone, 'output') == "ACCEPT" + assert u.get("firewall", zone, 'forward') == "REJECT" + assert u.get("firewall", forwardings[0]) == "forwarding" + assert u.get("firewall", forwardings[0], 'src') == "mytrusted" + assert u.get("firewall", forwardings[0], 'dest') == "lan" + assert u.get("firewall", forwardings[0], 'ns_tag') == "automated" + assert u.get("firewall", forwardings[1]) == "forwarding" + assert u.get("firewall", forwardings[1], 'src') == "lan" + assert u.get("firewall", forwardings[1], 'dest') == "mytrusted" + assert u.get("firewall", forwardings[1], 'ns_tag') == "automated" + assert u.get("firewall", forwardings[2]) == "forwarding" + assert u.get("firewall", forwardings[2], 'src') == "mytrusted" + assert u.get("firewall", forwardings[2], 'dest') == "wan" + assert u.get("firewall", forwardings[2], 'ns_tag') == "automated" + + link = 'openvpn/instance1' + (zone, forwardings) = firewall.add_trusted_zone(u, 'mylinked', link=link) + assert u.get("firewall", zone, 'ns_link') == link + assert u.get("firewall", forwardings[0], 'ns_link') == link + assert u.get("firewall", forwardings[1], 'ns_link') == link + assert u.get("firewall", forwardings[2], 'ns_link') == link + +def test_duplicated_add_trusted_zone(u): + (zone, forwardings) = firewall.add_trusted_zone(u, 'mytrusted') + assert zone is None + assert forwardings is None + + trusted = 0 + for s in u.get_all('firewall'): + if u.get('firewall', s) == 'forwarding': + if u.get('firewall', s, 'src', default='') == "mytrusted" and u.get('firewall', s, 'dest', default='') == "lan": + trusted = trusted + 1 + assert trusted == 1 + +def test_add_trusted_zone_with_networks(u): + interface = firewall.add_vpn_interface(u, 'testvpn2', 'tuntest2') + zone, forwardings = firewall.add_trusted_zone(u, 'mytrusted2', list(interface)) + assert u.get_all("firewall", zone, 'network') == tuple(interface) + +def test_apply(): + # Already tested in pyuci + assert 1 + +def test_add_template_rule(u): + rule = firewall.add_template_rule(u, 'ns_test_rule', 'tcp', '443', 'test1/key1') + assert u.get("firewall", rule) == "rule" + assert u.get("firewall", rule, "proto") == "tcp" + assert u.get("firewall", rule, "name") == "Test-rule" + assert u.get("firewall", rule, "src") == "wan" + assert u.get("firewall", rule, "dest") == "blue" + assert u.get("firewall", rule, "dest_port") == "443" + assert u.get("firewall", rule, "enabled") == "1" + assert u.get("firewall", rule, "target") == "ACCEPT" + assert u.get("firewall", rule, "ns_tag") == "automated" + assert u.get("firewall", rule, "ns_link") == "test1/key1" + + rule = firewall.add_template_rule(u, 'ip6_dhcp') + assert u.get("firewall", rule) == "rule" + assert u.get("firewall", rule, "proto") == "udp" + assert u.get("firewall", rule, "name") == "Allow-DHCPv6" + assert u.get("firewall", rule, "src") == "wan" + assert u.get("firewall", rule, "dest_port") == "546" + assert u.get("firewall", rule, "family") == "ipv6" + assert u.get("firewall", rule, "target") == "ACCEPT" + assert u.get("firewall", rule, "ns_tag") == "automated" + +def test_add_template_zone(u): + (zone, forwardings) = firewall.add_template_zone(u, 'ns_blue', ["lan", "lan2"], link="mydb/mykey" ) + assert zone is not None + assert u.get("firewall", zone) == "zone" + assert u.get("firewall", zone, "name") == "blue" + assert u.get("firewall", zone, "forward") == "DROP" + assert u.get("firewall", zone, "input") == "DROP" + assert u.get("firewall", zone, "output") == "ACCEPT" + assert u.get("firewall", zone, "ns_description") == "Guest network with Internet access" + assert u.get("firewall", zone, "ns_tag") == "automated" + assert u.get("firewall", zone, "ns_link") == "mydb/mykey" + assert "lan" in u.get("firewall", zone, "network", list=True) + assert "lan2" in u.get("firewall", zone, "network", list=True) + assert len(forwardings) == 2 + for f in forwardings: + assert u.get("firewall", f) == "forwarding" + assert u.get("firewall", f, "ns_tag") == "automated" + assert u.get("firewall", f, "src") == "blue" or u.get("firewall", f, "dest") == "blue" + assert u.get("firewall", f, "ns_link", default='') == "mydb/mykey" + (zone, forwardings) = firewall.add_template_zone(u, 'ns_blue') + assert zone is None + assert forwardings is None + +def test_add_template_service_group(u): + sections = firewall.add_template_service_group(u, "ns_web_secure") + assert len(sections) == 2 + assert u.get("firewall", sections[0]) == "rule" + assert u.get("firewall", sections[0], "src") == "lan" + assert u.get("firewall", sections[0], "dest") == "wan" + assert u.get("firewall", sections[0], "proto") == "tcp" + assert u.get("firewall", sections[0], "dest_port") == "80,443" + assert u.get("firewall", sections[0], "ns_tag") == "automated" + + assert u.get("firewall", sections[1]) == "rule" + assert u.get("firewall", sections[1], "src") == "lan" + assert u.get("firewall", sections[1], "dest") == "wan" + assert u.get("firewall", sections[1], "proto") == "udp" + assert u.get("firewall", sections[1], "dest_port") == "53" + assert u.get("firewall", sections[1], "ns_tag") == "automated" + + sections = firewall.add_template_service_group(u, "ns_web_secure", "grey", "orange") + assert u.get("firewall", sections[0], "src") == "grey" + assert u.get("firewall", sections[0], "dest") == "orange" + assert u.get("firewall", sections[0], "proto") == "tcp" + assert u.get("firewall", sections[1], "proto") == "udp" + + sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link="db/mykey") + assert u.get("firewall", sections[0], "ns_link") == "db/mykey" + assert u.get("firewall", sections[1], "ns_link") == "db/mykey" + +def test_get_all_linked(u): + link = "mytestdb/mykey" + sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) + rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) + interface = firewall.add_vpn_interface(u, 'p2p', 'ppp10', link=link) + (zone, forwardings) = firewall.add_trusted_zone(u, 'mylinked2', link=link) + linked = firewall.get_all_linked(u, link) + for s in sections: + assert s in linked['firewall'] + assert rule in linked['firewall'] + assert zone in linked['firewall'] + for f in forwardings: + assert f in linked['firewall'] + assert interface in linked['network'] + + +def test_disable_linked_rules(u): + link = "mytestdb/mykey" + sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) + rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) + interface = firewall.add_vpn_interface(u, 'p2p', 'ppp10', link=link) + (zone, forwardings) = firewall.add_trusted_zone(u, 'mylinked4', link=link) + disabled = firewall.disable_linked_rules(u, link) + for s in sections: + assert u.get("firewall", s, "enabled") == "0" + assert s in disabled + assert u.get("firewall", rule, "enabled") == "0" + assert rule in disabled + assert u.get("firewall", zone, "enabled", default="XX") == "XX" # option must not be set + assert u.get("network", interface, "enabled", default="XX") == "XX" # option must not be set + for f in forwardings: + assert u.get("network", f, "enabled", default="XX") == "XX" # option must not be set + +def test_delete_linked_sections(u): + link = "mytestdb/mykey" + sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) + rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) + interface = firewall.add_vpn_interface(u, 'p2p', 'ppp10', link=link) + (zone, forwardings) = firewall.add_trusted_zone(u, 'mylinked3', link=link) + deleted = firewall.delete_linked_sections(u, link) + assert len(deleted) > 0 + with pytest.raises(UciExceptionNotFound): + u.get("firewall", rule) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", zone) + with pytest.raises(UciExceptionNotFound): + for s in sections: + u.get("firewall", s) + with pytest.raises(UciExceptionNotFound): + for f in forwardings: + u.get("firewall", f) + +def test_is_ipv6_enabled(u): + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'lan6') + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'wan6') + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'vlan6') + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'wan6b') + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'wan6c') + assert firewall.is_ipv6_enabled(u) == False + +def test_disable_ipv6_firewall(u): + assert u.get("firewall", "v6rule", "enabled", default="1") == "1" + firewall.disable_ipv6_firewall(u) + assert u.get("firewall", "v6rule", "enabled", default="1") == "0" + + +def test_list_zones(u): + assert firewall.list_zones(u)["ns_lan"]["name"] == "lan" + assert firewall.list_zones(u)["ns_lan"]["input"] == "ACCEPT" + assert firewall.list_zones(u)["ns_lan"]["output"] == "ACCEPT" + assert firewall.list_zones(u)["ns_lan"]["forward"] == "ACCEPT" + assert firewall.list_zones(u)["ns_lan"]["network"] == ("GREEN_1", "al_lan") + assert firewall.list_zones(u)["ns_wan"]["name"] == "wan" + assert firewall.list_zones(u)["ns_wan"]["input"] == "REJECT" + assert firewall.list_zones(u)["ns_wan"]["output"] == "ACCEPT" + assert firewall.list_zones(u)["ns_wan"]["forward"] == "REJECT" + assert firewall.list_zones(u)["ns_wan"]["network"] == ("wan6", "RED_2", "RED_3", "RED_1") + assert firewall.list_zones(u)["ns_wan"]["log"] == "1" + + +def list_zones_no_aliases(u): + assert firewall.list_zones_no_aliases(u)["ns_lan"]["network"] == ("GREEN_1",) + + +def test_list_forwardings(u): + assert firewall.list_forwardings(u)["ns_lan2guests"]["src"] == "lan" + assert firewall.list_forwardings(u)["ns_lan2guests"]["dest"] == "guests" + assert firewall.list_forwardings(u)["ns_guests2wan"]["src"] == "guests" + assert firewall.list_forwardings(u)["ns_guests2wan"]["dest"] == "wan" + + +def test_add_zone(u): + assert firewall.add_zone(u, "new_zone", "REJECT", "DROP", True, ["lan"], ["lan", "guest"]) == ( + "ns_new_zone", {"ns_new_zone2wan", "ns_new_zone2lan", "ns_lan2new_zone", "ns_guest2new_zone"}) + assert u.get("firewall", "ns_new_zone", "name") == "new_zone" + assert u.get("firewall", "ns_new_zone", "input") == "REJECT" + assert u.get("firewall", "ns_new_zone", "output") == "ACCEPT" + assert u.get("firewall", "ns_new_zone", "forward") == "DROP" + assert u.get("firewall", "ns_new_zone", "log") == "0" + assert u.get("firewall", "ns_new_zone2wan", "src") == "new_zone" + assert u.get("firewall", "ns_new_zone2wan", "dest") == "wan" + assert u.get("firewall", "ns_new_zone2lan", "src") == "new_zone" + assert u.get("firewall", "ns_new_zone2lan", "dest") == "lan" + assert u.get("firewall", "ns_lan2new_zone", "src") == "lan" + assert u.get("firewall", "ns_lan2new_zone", "dest") == "new_zone" + assert u.get("firewall", "ns_guest2new_zone", "src") == "guest" + assert u.get("firewall", "ns_guest2new_zone", "dest") == "new_zone" + assert firewall.add_zone(u, "new_zone_with_log", "REJECT", "DROP", True, ["lan"], ["lan", "guest"], True) + assert u.get("firewall", "ns_new_zone_with_log", "log") == "1" + assert u.get("firewall", "ns_new_zone_with_log", "log_limit") == "5/s" + +def test_edit_zone(u): + assert firewall.edit_zone(u, "new_zone", "DROP", "ACCEPT", False, ["lan"], ["lan", "guest"]) == ( + "ns_new_zone", {"ns_new_zone2lan", "ns_lan2new_zone", "ns_guest2new_zone"}) + assert u.get("firewall", "ns_new_zone", "name") == "new_zone" + assert u.get("firewall", "ns_new_zone", "input") == "DROP" + assert u.get("firewall", "ns_new_zone", "output") == "ACCEPT" + assert u.get("firewall", "ns_new_zone", "forward") == "ACCEPT" + assert u.get("firewall", "ns_new_zone2lan", "src") == "new_zone" + assert u.get("firewall", "ns_new_zone2lan", "dest") == "lan" + assert u.get("firewall", "ns_lan2new_zone", "src") == "lan" + assert u.get("firewall", "ns_lan2new_zone", "dest") == "new_zone" + assert u.get("firewall", "ns_guest2new_zone", "src") == "guest" + assert u.get("firewall", "ns_guest2new_zone", "dest") == "new_zone" + assert u.get("firewall", "ns_new_zone", "log") == "0" + assert firewall.edit_zone(u, "new_zone", "DROP", "ACCEPT", False, ["lan"], ["lan", "guest"], True) + assert u.get("firewall", "ns_new_zone", "log") == "1" + + +def test_delete_zone(u): + assert firewall.delete_zone(u, "ns_new_zone") == ( + "ns_new_zone", {"ns_new_zone2lan", "ns_guest2new_zone", "ns_lan2new_zone"}) + with pytest.raises(Exception) as e: + firewall.delete_zone(u, "not_a_zone") + +def test_get_rule_by_name(u): + assert firewall.get_rule_by_name(u, "Allow-DHCPv6") == ( + "v6rule", + {"name": "Allow-DHCPv6", "src": "wan", "proto": "udp", "dest_port": "546", "family": "ipv6", "target": "ACCEPT", "enabled": "0"} + ) + assert firewall.get_rule_by_name(u, "Not-Automated") == ( + "manrule", + {"name": "Not-Automated", "src": "wan", "proto": "tcp", "dest_port": "1234", "family": "ipv4", "target": "ACCEPT"} + ) + assert firewall.get_rule_by_name(u, "not_a_rule") == (None, None) + assert firewall.get_rule_by_name(u, "Not-Automated", "automated") == (None, None) + +def test_add_default_ipv6_rules(u): + # one rule should be skipped because it already exists + assert len(firewall.add_default_ipv6_rules(u)) == 3 + assert firewall.add_default_ipv6_rules(u) == [] + +def test_resolve_address(u): + assert firewall.resolve_address(u, "192.168.100.1") == {"value": "192.168.100.1", "type": "domain", "label": "test.name.org"} + assert firewall.resolve_address(u, "192.168.100.2") == {"value": "192.168.100.2", "type": "host", "label": "test2.giacomo.org"} + assert firewall.resolve_address(u, "192.168.100.238") == {"value": "192.168.100.238", "type": "interface", "label": "lan"} + assert firewall.resolve_address(u, "ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543") == {"value": "ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543", "type": "domain", "label": "test3.test.org"} + assert firewall.resolve_address(u, "10.0.0.22") == {"value": "10.0.0.22", "type": "interface", "label": "bond1"} + assert firewall.resolve_address(u, "2001:db80::2/64") == {"value": "2001:db80::2/64", "type": "interface", "label": "wan6"} + +def test_is_zone(u): + assert firewall.is_zone(u,"lan") == True + assert firewall.is_zone(u,"wan") == True + assert firewall.is_zone(u,"blue") == True + assert firewall.is_zone(u,"wan") == True + # we expect it fails because the zone does not exist + assert firewall.is_zone(u,"lan2custom") == False + # we expect it to be true because we are using any zone + assert firewall.is_zone(u,"*") == True + +def test_list_forward_rules(u): + rules = firewall.list_forward_rules(u) + assert len(rules) > 0 + for r in rules: + # just check that only forwarded rules are returned + assert r.get("src") and r.get("dest") + # check all fields are presents + assert 'log' in r + assert 'enabled' in r + assert 'ns_tag' in r + assert 'proto' in r + assert 'active_zone' in r + +def test_list_output_rules(u): + rules = firewall.list_output_rules(u) + assert len(rules) > 0 + for r in rules: + # just check that only output rules are returned + assert r.get("src") is None and r.get("dest") + # check all fields are presents + assert 'log' in r + assert 'enabled' in r + assert 'ns_tag' in r + assert 'proto' in r + assert 'active_zone' in r + +def test_list_input_rules(u): + rules = firewall.list_input_rules(u) + assert len(rules) > 0 + for r in rules: + # just check that only input rules are returned + assert r.get("dest") is None and r.get("src") + # check all fields are presents + assert 'log' in r + assert 'enabled' in r + assert 'ns_tag' in r + assert 'proto' in r + assert 'active_zone' in r + +def test_list_service_suggestions(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + services = firewall.list_service_suggestions() + assert len(services) == 5 + assert services == [{'id': 'ftp', 'proto': ['tcp'], 'port': 21}, {'id': 'ssh', 'proto': ['tcp', 'udp'], 'port': 22}, {'id': 'time', 'proto': ['udp'], 'port': 37}, {'id': 'www', 'proto': ['tcp'], 'port': 80}, {'id': 'kerberos', 'proto': ['tcp', 'udp'], 'port': 88}] + +def test_list_host_suggestions(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=lease_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + suggestions = firewall.list_host_suggestions(u) + assert len(suggestions) == 10 + assert suggestions == [ + {'value': '192.168.100.1', 'label': 'test.name.org', 'type': 'domain'}, + {'value': '192.168.100.2', 'label': 'test2.giacomo.org', 'type': 'host'}, + {'value': 'ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543', 'label': 'test3.test.org', 'type': 'domain'}, + {'value': '7.8.9.1', 'label': 'host1', 'type': 'domain'}, + {'value': '192.168.100.238', 'label': 'lan', 'type': 'network'}, + {'value': '2001:db80::2/64', 'label': 'wan6', 'type': 'network'}, + {'value': '10.0.0.22', 'label': 'bond1', 'type': 'network'}, + {'value': '10.0.0.1/24', 'label': 'al_lan', 'type': 'network'}, + {'value': '192.168.1.228', 'label': 'test1', 'type': 'lease'}, + {'value': '192.168.1.219', 'label': 'test2', 'type': 'lease'}, + ] + +def test_add_rule(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + rid = firewall.add_rule(u, 'myrule', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "ssh", True, True, ['tag1'], False) + assert u.get("firewall", rid, "name") == "myrule" + assert u.get("firewall", rid, "src") == "lan" + assert u.get("firewall", rid, "dest") == "wan" + assert u.get_all("firewall", rid, "src_ip") == ("192.168.1.22",) + assert u.get_all("firewall", rid, "dest_ip") == ("1.2.3.4",) + assert u.get_all("firewall", rid, "proto") == ('tcp', 'udp') + assert u.get("firewall", rid, "dest_port") == "22" + assert u.get("firewall", rid, "target") == "ACCEPT" + assert u.get("firewall", rid, "ns_service") == "ssh" + assert u.get("firewall", rid, "enabled") == "1" + assert u.get("firewall", rid, "log") == "1" + assert u.get("firewall", rid, "log_limit") == "1/s" + assert u.get_all("firewall", rid, "ns_tag") == ("tag1",) + assert u.get("firewall", rid, "ns_link", default="notpresent") == "notpresent" + +def test_edit_rule(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + rid = firewall.add_rule(u, 'myrule2', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '22', 'ACCEPT', "ssh", True, True, ['tag1'], False) + with pytest.raises(ValidationError): + firewall.edit_rule(u, 'notpresent', 'myrule3', 'lan', [], 'wan', [], [], '', 'ACCEPT', None, True, True, ['tag1']) + rid2 = firewall.edit_rule(u, rid, 'myrule3', 'lan', [], 'blue', [], ['udp'], '22', 'DROP', "", False, False, ['tag2']) + assert rid == rid2 + assert u.get("firewall", rid, "name") == "myrule3" + assert u.get("firewall", rid, "src") == "lan" + assert u.get("firewall", rid, "dest") == "blue" + assert u.get_all("firewall", rid, "proto") == "all" + assert u.get("firewall", rid, "ns_service") == "*" + with pytest.raises(UciExceptionNotFound): + u.get_all("firewall", rid, "src_ip") + with pytest.raises(UciExceptionNotFound): + u.get_all("firewall", rid, "dest_ip") + with pytest.raises(UciExceptionNotFound): + u.get("firewall", rid, "dest_port") == "22" + assert u.get("firewall", rid, "target") == "DROP" + assert u.get("firewall", rid, "enabled") == "0" + assert u.get("firewall", rid, "log") == "0" + assert u.get_all("firewall", rid, "ns_tag") == ("tag2",) + rid3 = firewall.edit_rule(u, rid, 'myrule3', 'lan', [], 'blue', [], ['udp'], '22', 'DROP', "www", False, False, ['tag2']) + assert u.get("firewall", rid, "ns_service") == "www" + assert u.get_all("firewall", rid, "proto") == ('tcp',) + assert u.get("firewall", rid, "dest_port") == "80" + +def test_add_rule_with_notrack(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + rid = firewall.add_rule(u, 'notrack_rule', 'lan', ['192.168.1.0/24'], 'wan', [], [], '', 'NOTRACK', "*", True, False, [], False) + assert u.get("firewall", rid, "name") == "notrack_rule" + assert u.get("firewall", rid, "target") == "NOTRACK" + assert u.get("firewall", rid, "src") == "lan" + assert u.get("firewall", rid, "dest") == "wan" + assert u.get_all("firewall", rid, "src_ip") == ("192.168.1.0/24",) + assert u.get("firewall", rid, "enabled") == "1" + +def test_edit_rule_to_notrack(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + rid = firewall.add_rule(u, 'rule_to_change', 'lan', [], 'wan', [], [], '', 'ACCEPT', "*", True, False, [], False) + assert u.get("firewall", rid, "target") == "ACCEPT" + firewall.edit_rule(u, rid, 'rule_to_change', 'lan', [], 'wan', [], [], '', 'NOTRACK', "*", True, False, []) + assert u.get("firewall", rid, "target") == "NOTRACK" + +def test_delete_rule(u): + ids = firewall.list_rule_ids(u) + id_to_delete = ids.pop() + firewall.delete_rule(u, id_to_delete) + assert id_to_delete not in firewall.list_rule_ids(u) + +def test_disable_rule(u): + ids = firewall.list_rule_ids(u) + id_to_disable = ids.pop() + firewall.disable_rule(u, id_to_disable) + assert u.get("firewall", id_to_disable, "enabled") == "0" + +def test_enable_rule(u): + ids = firewall.list_rule_ids(u) + id_to_enable = ids.pop() + firewall.disable_rule(u, id_to_enable) + assert u.get("firewall", id_to_enable, "enabled") == "0" + firewall.enable_rule(u, id_to_enable) + assert u.get("firewall", id_to_enable, "enabled") == "1" + +def test_order_rules(u, mocker): + # The firewall.order_rules function uses the uci binary to reorder the rules + # The test is usefull because uci behaves differently on a real machine + assert True + +def test_list_nat_rules(u): + names = [] + for r in firewall.list_nat_rules(u): + names.append(r.get("name")) + assert "source_NAT1_vpm" in names + assert "masquerade" in names + assert "cdn_via_router" in names + assert "SNAT_NSEC7_style" in names + assert "Allow-DHCPv6" not in names + +def test_add_nat_rule(u): + id1 = firewall.add_nat_rule(u, "myrule", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") + assert u.get("firewall", id1, "name") == "myrule" + assert u.get("firewall", id1, "target") == "SNAT" + assert u.get("firewall", id1, "src") == "lan" + assert u.get("firewall", id1, "src_ip") == "1.2.3.4" + assert u.get("firewall", id1, "dest_ip") == "6.7.8.9" + assert u.get_all("firewall", id1, "proto") == ("all",) + with pytest.raises(ValidationError): + firewall.add_nat_rule(u, "myrule", "REJECT") + id2 = firewall.add_nat_rule(u, "myrule3", "ACCEPT", dest_ip="1.2.3.4") + assert u.get("firewall", id2, "target") == "ACCEPT" + assert u.get("firewall", id2, "dest_ip") == "1.2.3.4" + assert u.get_all("firewall", id2, "proto") == ("all",) + assert u.get("firewall", id2, "src") == "*" + with pytest.raises(UciExceptionNotFound): + u.get("firewall", id2, "src_ip") + +def test_edit_nat_rule(u): + id = firewall.add_nat_rule(u, "myrule4", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") + firewall.edit_nat_rule(u, id, "myrule4b", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "3.3.3.3") + assert u.get("firewall", id, "name") == "myrule4b" + assert u.get("firewall", id, "snat_ip") == "3.3.3.3" + assert u.get("firewall", id, "src_ip") == "1.2.3.4" + assert u.get("firewall", id, "dest_ip") == "6.7.8.9" + assert u.get("firewall", id, "target") == "SNAT" + assert u.get_all("firewall", id, "proto") == ("all",) + assert u.get("firewall", id, "src") == "lan" + +def test_delete_nat_rule(u): + with pytest.raises(ValidationError): + firewall.delete_nat_rule(u, "notpresent") + id = firewall.add_nat_rule(u, "myrule5", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") + assert firewall.delete_nat_rule(u, id) == id + +def test_list_netmap_rules(u): + names = [] + for r in firewall.list_netmap_rules(u): + names.append(r.get("name")) + assert "source_nat1" in names + assert "dest_nat1" in names + +def test_add_netmap_rule(u): + with pytest.raises(ValidationError): + id1 = firewall.add_netmap_rule(u, "myrule", "10.50.51.0/24", "10.50.50.0/24", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") + id1 = firewall.add_netmap_rule(u, "myrule", "10.50.51.0/24", "", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") + assert u.get("netmap", id1, "name") == "myrule" + assert u.get("netmap", id1, "src") == "10.50.51.0/24" + assert u.get_all("netmap", id1, "device_in") == ("eth0",) + assert u.get_all("netmap", id1, "device_out") == ("eth1",) + assert u.get("netmap", id1, "map_from") == "10.10.10.0/24" + assert u.get("netmap", id1, "map_to") == "192.168.1.0/24" + id2 = firewall.add_netmap_rule(u, "myrule2", "", "10.50.50.0/24", ["eth0"], [], "10.10.10.0/24", "192.168.1.0/24") + assert u.get("netmap", id2, "name") == "myrule2" + with pytest.raises(UciExceptionNotFound): + u.get("netmap", id2, "src") + assert u.get("netmap", id2, "dest") == "10.50.50.0/24" + assert u.get_all("netmap", id2, "device_in") == ("eth0",) + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id2, "device_out") + assert u.get("netmap", id2, "map_from") == "10.10.10.0/24" + assert u.get("netmap", id2, "map_to") == "192.168.1.0/24" + +def test_edit_netmap_rule(u): + id = firewall.add_netmap_rule(u, "myrule3", "", "10.50.50.0/24", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") + firewall.edit_netmap_rule(u, id, "myrule3b", "", "10.50.51.0/24", [], [], "10.10.11.0/24", "192.168.2.0/24") + assert u.get("netmap", id, "name") == "myrule3b" + with pytest.raises(UciExceptionNotFound): + u.get("netmap", id, "src") + assert u.get("netmap", id, "dest") == "10.50.51.0/24" + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id, "device_out") + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id, "device_in") + assert u.get("netmap", id, "map_from") == "10.10.11.0/24" + assert u.get("netmap", id, "map_to") == "192.168.2.0/24" + +def test_delete_netmap_rule(u): + with pytest.raises(ValidationError): + firewall.delete_netmap_rule(u, "notpresent") + id = firewall.add_netmap_rule(u, "myrule3b", "", "10.50.51.0/24", None, None, "10.10.11.0/24", "192.168.2.0/24") + assert firewall.delete_netmap_rule(u, id) == id + +@patch("nethsec.utils.subprocess.run") +def test_list_netmap_devices(mock_run, u): + u.get_all("netmap", id, "device_out") + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id, "device_in") + assert u.get("netmap", id, "map_from") == "10.10.11.0/24" + assert u.get("netmap", id, "map_to") == "192.168.2.0/24" + +def test_delete_netmap_rule(u): + with pytest.raises(ValidationError): + firewall.delete_netmap_rule(u, "notpresent") + id = firewall.add_netmap_rule(u, "myrule3b", "", "10.50.51.0/24", None, None, "10.10.11.0/24", "192.168.2.0/24") + assert firewall.delete_netmap_rule(u, id) == id + +@patch("nethsec.utils.subprocess.run") +def test_list_netmap_devices(mock_run, u): + mock_run.return_value = mock_ip_stdout + devices = firewall.list_netmap_devices(u) + device_names = [d.get("device") for d in devices] + assert 'lo' not in device_names + assert 'br-lan' in device_names + assert 'eth1.4' in device_names + assert {'device': 'br-lan', 'interface': 'lan'} in devices + +def test_update_redirect_rules(u): + domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) + ipsets = objects.get_domain_set_ipsets(u, domain1) + host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) + u.set("firewall", "redirect3", "ns_src", f"objects/{domain1}") # domain can be used only as source + u.set('firewall', 'redirect3', 'ns_dst', f"dhcp/ns_dhcp1") + firewall.update_redirect_rules(u) + print(u.get_all("firewall", "redirect3")) + assert u.get("firewall", "redirect3", "dest_ip") == "7.8.9.1" + assert u.get("firewall", "redirect3", "ipset") == f"{ipsets['firewall']} src_net" + u.set("firewall", "redirect4", "ns_src", f"objects/{host1}") + u.set('firewall', 'redirect4', 'ns_dst', f"users/ns_user1") + firewall.update_redirect_rules(u) + assert u.get("firewall", "redirect4", "dest_ip") == "10.10.10.22" + assert u.get("firewall", "redirect4", "ipset") == f"redirect4_ipset" + assert u.get("firewall", "redirect4_ipset") + assert u.get('firewall', 'redirect4_ipset', 'ns_link') == f'firewall/redirect4' + +def test_update_firewall_rules(u): + domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) + ipsets = objects.get_domain_set_ipsets(u, domain1) + host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) + u.set("firewall", "r1", "ns_dst", f"objects/{domain1}") + u.set("firewall", "r1", "ns_src", f"objects/{host1}") + firewall.update_firewall_rules(u) + assert set(u.get_all("firewall", "r1", "src_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) + assert u.get("firewall", "r1", "ipset") == f"{ipsets['firewall']} dst" + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r1", "dest_ip") + + u.set("firewall", "r2", "ns_dst", f"objects/{host1}") + u.set("firewall", "r2", "ns_src", f"objects/{domain1}") + firewall.update_firewall_rules(u) + + assert u.get("firewall", "r2", "ipset") == f"{ipsets['firewall']} src" + assert set(u.get_all("firewall", "r2", "dest_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r2", "src_ip") + + u.set("firewall", "r3", "ns_src", "dhcp/ns_dhcp1") + u.set("firewall", "r3", "ns_dst", "users/ns_user1") + firewall.update_firewall_rules(u) + assert u.get_all("firewall", "r3", "src_ip") == ("7.8.9.1",) + assert u.get_all("firewall", "r3", "dest_ip") == ("10.10.10.22",) + + u.set("firewall", "r4", "ns_src", f"objects/{domain1}") + u.set("firewall", "r4", "dest_ip", "1.2.3.4") + firewall.update_firewall_rules(u) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r4", "ns_dst") + +def test_update_firewall_rules_concurrency(u): + host1 = objects.add_host_set(u, "hc1", "ipv4", ["1.1.1.1"]) + host2 = objects.add_host_set(u, "hc2", "ipv4", ["2.2.2.2"]) + rid = firewall.add_rule(u, 'mycrule', '*', [], 'wan', [], [], '', 'DROP', "", True, False, ['tag1'], False, f"objects/{host1}", f"objects/{host2}") + assert u.get("firewall", rid, "src_ip") == "1.1.1.1" + assert u.get("firewall", rid, "dest_ip") == "2.2.2.2" + firewall.update_firewall_rules(u) + firewall.update_redirect_rules(u) + assert u.get("firewall", rid, "src_ip") == "1.1.1.1" + assert u.get("firewall", rid, "dest_ip") == "2.2.2.2" + firewall.delete_rule(u, rid) + objects.delete_host_set(u, host1) + objects.delete_host_set(u, host2) + +def test_list_object_suggestions(u): + obj = objects.list_objects(u) + assert len(obj) == 9 + +def test_edit_rule_remove_object(u): + host1 = objects.add_host_set(u, "h1", "ipv4", ["1.2.3.4"]) + # def add_rule(uci, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled=True, log=False, tag=[], add_to_top=False, ns_src=None, ns_dst=None): + idf1 = firewall.add_rule(u, "forward1", "*", [], "wan", [], [], [], "REJECT", "*", True, False, [], False, f"objects/{host1}", "") + # remove object from rule + firewall.edit_rule(u, idf1, "forward1", "*", [], "wan", [], [], [], "REJECT", "*", True, False, []) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", idf1, "src_ip") + assert u.get('firewall', idf1, 'ns_src', default='NONE') == 'NONE' + firewall.delete_rule(u, idf1) + objects.delete_host_set(u, host1) + +def test_get_default_logging_options(u): + logging_options = firewall.get_default_logging_options(u) + assert logging_options['rule_log_limit'] == '1/s' + assert logging_options['zone_log_limit'] == '5/s' + assert logging_options['redirect_log_limit'] == '1/s' + +def test_rule_default_log_limit(u): + rid = firewall.add_rule(u, 'myrule3', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "", log=True) + assert u.get("firewall", rid, "log_limit") == "1/s" + +def test_change_rule_default_log_limit(u): + u.set('firewall', 'ns_defaults', 'rule_log_limit', '2/s') + u.save('firewall') + rid = firewall.add_rule(u, 'myrule4', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "", log=True) + assert u.get("firewall", rid, "log_limit") == "2/s" + +def test_assert_custom_log_limit(u): + # Check rule + rid = firewall.add_rule(u, 'myrule5', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "", log=True) + u.set('firewall', rid, 'log_limit', '123/s') + u.save('firewall') + rid = firewall.edit_rule(u, rid, 'myrule5', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "", log=True) + assert u.get("firewall", rid, "log_limit") == "123/s" + # Check zone + (zid, forwardings) = firewall.add_zone(u, "ztwl", "REJECT", "DROP", log=True) + print(zid) + u.set('firewall', zid, 'log_limit', '123/s') + u.save('firewall') + firewall.edit_zone(u, zid, "DROP", "ACCEPT", log=True) + assert u.get('firewall', zid, 'log_limit') == '123/s' + +def test_change_zone_default_log_limit(u): + u.set("firewall", "ns_defaults", "zone_log_limit", "20/s") + u.save('firewall') + (zid, forwardings) = firewall.add_zone(u, "new_zone3", "REJECT", "DROP", True, ["lan"], ["lan", "guest"], True) + assert u.get("firewall", zid, "log_limit") == "20/s" + +def test_apply_default_logging_options(u): + firewall.apply_default_logging_options(u) + assert u.get("firewall", "ns_59eea167", "log_limit") == "1/s" + assert u.get("firewall", "f1", "log_limit") == "2/s" + assert u.get("firewall", "with_log", "log_limit") == "20/s" + assert u.get("firewall", "grey", "log_limit", default=None) == None + assert u.get("firewall", "o1", "log_limit", default=None) == None + assert u.get("firewall", "redirect3", "log_limit", default=None) == None + + +def test_ns_link_rules(u): + rid = firewall.add_rule(u, 'ns_link_rule', 'lan', [], 'wan', [], [], [], 'ACCEPT', '', ns_link='test/example') + assert u.get('firewall', rid, 'ns_link') == 'test/example' + firewall.edit_rule(u, rid, 'ns_link_rule', 'lan', [], 'wan', [], [], [], 'ACCEPT', '', ns_link='test/example2') + assert u.get('firewall', rid, 'ns_link') == 'test/example2' + firewall.edit_rule(u, rid, 'ns_link_rule', 'lan', [], 'wan', [], [], [], 'ACCEPT', '') + assert u.get('firewall', rid, 'ns_link', default=None) is None + rid = firewall.add_rule(u, 'ns_link_rule_no_link', 'lan', [], 'wan', [], [], [], 'ACCEPT', '') + assert u.get('firewall', rid, 'ns_link', default=None) is None \ No newline at end of file diff --git a/packages/python3-nethsec/tests/test_inventory.py b/packages/python3-nethsec/tests/test_inventory.py new file mode 100644 index 000000000..c81ab7eee --- /dev/null +++ b/packages/python3-nethsec/tests/test_inventory.py @@ -0,0 +1,1324 @@ +from euci import EUci +from unittest.mock import patch, mock_open, MagicMock +import subprocess + +from nethsec import inventory + +netmap_db = """ + +config rule 'ns_b606fef5' + option name 'foo' + option dest '8.8.8.0/24' + option map_from '12.0.0.0/24' + option map_to '16.0.0.0/24' + +config rule 'ns_003ff675' + option name 'john' + option src '12.0.0.0/24' + option map_from '15.0.0.0/24' + option map_to '14.0.0.0/24' +""" + +firewall_db = """ +config zone lan1 + option name 'lan' + list network 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list device 'eth0' + +config zone grey + option name 'grey' + list network 'grey1' + option input 'DROP' + option output 'ACCEPT' + option forward 'DROP' + list device 'eth1' + +config zone orange + option name 'orange' + list network 'orange1' + option input 'DROP' + option output 'DROP' + option forward 'DROP' + +config zone wan1f + option name 'wan' + list network 'wan' + list network 'wan6' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + +config forwarding fw1 + option src 'lan' + option dest 'wan' + +config rule 'v6rule' + option name 'Allow-DHCPv6' + option src 'wan' + option proto 'udp' + option dest_port '546' + option family 'ipv6' + option target 'ACCEPT' + +config rule 'manrule' + option name 'Not-Automated' + option src 'wan' + option proto 'tcp' + option dest_port '1234' + option family 'ipv4' + option target 'ACCEPT' + +config rule 'f1' + option name 'r1' + option dest 'wan' + option dest_port '22' + option target 'ACCEPT' + option src 'lan' + list src_ip '192.168.100.1' + list src_ip '192.168.100.238' + list dest_ip '192.168.122.1' + list dest_ip '192.168.122.49' + option log '1' + +config rule 'o1' + option name 'output1' + list dest_ip '192.168.100.1' + option target 'ACCEPT' + option dest 'wan' + +config rule 'i1' + option name 'Allow-OpenVPNRW1' + option src 'wan' + option dest_port '1194' + option proto 'udp' + option target 'ACCEPT' + list ns_tag 'automated' + option ns_link 'openvpn/ns_roadwarrior1' + +config nat + option name 'source_NAT1_vpm' + option src '*' + option src_ip '192.168.55.0/24' + option dest_ip '10.20.30.0/24' + option target 'SNAT' + option snat_ip '10.44.44.1' + list proto 'all' + +config nat + option name 'masquerade' + list proto 'all' + option src 'lan' + option src_ip '192.168.1.0/24' + option dest_ip '10.88.88.0/24' + option target 'MASQUERADE' + +config nat + option name 'cdn_via_router' + list proto 'all' + option src 'lan' + option src_ip '192.168.1.0/24' + option dest_ip '192.168.50.0/24' + option target 'ACCEPT' + +config nat + option name 'SNAT_NSEC7_style' + list proto 'all' + option src 'wan' + option src_ip '192.168.1.44' + option target 'SNAT' + option snat_ip '10.20.30.5' + +config redirect 'redirect3' + option ns_src '' + option ipset 'redirect3_ipset' + +config ipset 'redirect3_ipset' + option name 'redirect1_ipset' + option match 'src_net' + option enabled '1' + list entry '6.7.8.9' + +config redirect 'redirect4' + option ns_src '' + +config rule 'r1' + option name 'r1' + option ns_dst 'dhcp/ns_8dcab636' + +config rule 'r2' + option name 'r2' + option ns_dst '' + +config rule 'r3' + option name 'r3' + option ns_src '' + +config rule 'r4' + option ns_dst '' + option ns_src '' + +config rule 'ns_72004856' + option name 'forward' + option src '*' + option dest 'lan' + option target 'DROP' + option ns_service '*' + option proto 'all' + option enabled '1' + option log '0' + list src_ip '1.1.1.1' + option ns_dst 'objects/ns_7c9c0852' + option ipset 'foo dst' + +config rule 'ns_df2ca87c' + option name 'output' + option dest '*' + option target 'DROP' + option ns_service 'discard' + option dest_port '9' + option enabled '1' + option log '0' + list proto 'tcp' + list proto 'udp' + option ns_dst 'dhcp/ns_8d5d2cf4' + list dest_ip '192.168.50.123' + +config rule 'ns_a50bcca9' + option name 'input' + option src '*' + option target 'DROP' + option ns_service '*' + option proto 'all' + option enabled '1' + option log '0' + option ns_src 'objects/ns_bcfb447b' + list src_ip '192.168.50.123' + +""" + +network_db = """ +config interface 'lan' + option device 'br-lan' + option proto 'static' + option ipaddr '192.168.50.1' + option netmask '255.255.255.0' + +config interface 'wan' + option device 'eth1' + option proto 'dhcp' + +config interface 'wan6' + option device 'eth1' + option proto 'dhcpv6' + +config interface 'bond1' + option proto 'bonding' + option ipaddr '10.0.0.22' + option netmask '255.255.255.0' + list slaves 'eth3' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + option all_slaves_active '0' + option link_monitoring 'off' + +config interface 'bond2' + option proto 'bonding' + option ipaddr '10.11.11.22' + option netmask '255.255.255.0' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + list slaves 'eth3.2' + +config device 'ns_3b3556cf' + option name 'eth0.6' + option type '8021q' + option ifname 'eth0' + option vid '6' + option ipv6 '0' +config interface 'vlan6' + option device 'eth0.6' + option proto 'dhcp' + +config device 'ns_c3c5206a' + option name 'bond-foo' + option ipv6 '0' +config interface 'bond1' + option device 'bond-toto' + option proto 'static' + option ipaddr '125.12.12.12' + option netmask '255.255.255.0' +config route 'ns_31de6ad0' + option disabled '0' + option gateway '1.1.1.1' + option metric '0' + option mtu '1500' + option ns_description 'truc' + option onlink '0' + option target '1.1.1.0/24' + option type 'unicast' + +config route6 'ns_4a936870' + option disabled '0' + option gateway 'fe80::fc54:ff:fe70:2' + option metric '0' + option mtu '1500' + option ns_description 'peolopopo' + option onlink '0' + option target 'fe80::fc54:ff:fe70:d2bc/64' + option type 'unicast' + +""" + +dedalo_db = """ +config dedalo 'config' + option aaa_url 'https://my.nethspot.com/wax/aaa' + option api_url 'https://my.nethspot.com/api' + option network '192.168.82.0/24' + option disabled '0' + option splash_page 'http://my.nethspot.com/wings' + option max_clients '253' + option dhcp_end '250' + option dhcp_start '2' + option hotspot_id '51' + option unit_name 'fw1.nethsecurity.org' + option unit_description 'fw1 sede' + option interface 'eth2.10' + option secret 'xxxxxx' + option unit_uuid 'yyyyy' +""" + +flashstart_db = """ +config main 'global' + option enabled '0' + option proplus '1' + option password 'xxx' + option username 'xxx@nethsecurity.org' + list zones 'lan' + list bypass '192.168.1.252' + list bypass '192.168.1.211' + list custom_servers '/www.google.com/1.1.1.1' + """ + +openvpn_db = """ +config openvpn 'sample_client' + option client '1' + option dev 'tun' + option proto 'udp' + list remote 'my_server_1 1194' + option resolv_retry 'infinite' + option nobind '1' + option persist_key '1' + option persist_tun '1' + option user 'nobody' + option ca '/etc/openvpn/ca.crt' + option cert '/etc/openvpn/client.crt' + option key '/etc/openvpn/client.key' + option verb '3' + +config openvpn 'ns_openvpn1' + option dev 'tunopenvpn1' + option dev_type 'tun' + option enabled '1' + option persist_tun '1' + option float '1' + option multihome '1' + option passtos '1' + option ping_timer_rem '1' + option persist_key '1' + option keepalive '10 60' + option lport '1202' + option proto 'udp' + option topology 'subnet' + option dh '/etc/openvpn/ns_openvpn1/pki/dh.pem' + option ca '/etc/openvpn/ns_openvpn1/pki/ca.crt' + option cert '/etc/openvpn/ns_openvpn1/pki/issued/server.crt' + option key '/etc/openvpn/ns_openvpn1/pki/private/server.key' + option server '10.7.57.0 255.255.255.0' + list push 'topology subnet' + list push 'route 192.168.50.0 255.255.255.0' + list route '192.168.10.0 255.255.255.0' + option cipher 'AES-256-GCM' + option auth 'SHA256' + list ns_public_ip '86.206.129.103' + option ns_name 'openvpn1' + option client_connect '"/usr/libexec/ns-openvpn/openvpn-connect ns_openvpn1"' + option client_disconnect '"/usr/libexec/ns-openvpn/openvpn-disconnect ns_openvpn1"' + option management '/var/run/openvpn_ns_openvpn1.socket unix' + +config openvpn 'ns_roadwarrior1' + option proto 'udp' + option port '1194' + option dev 'tunrw1' + option dev_type 'tun' + option topology 'subnet' + option float '1' + option passtos '1' + option multihome '1' + option verb '3' + option enabled '1' + option keepalive '20 120' + option server '10.9.9.0 255.255.255.0' + option client_connect '"/usr/libexec/ns-openvpn/openvpn-connect ns_roadwarrior1"' + option client_disconnect '"/usr/libexec/ns-openvpn/openvpn-disconnect ns_roadwarrior1"' + option dh '/etc/openvpn/ns_roadwarrior1/pki/dh.pem' + option ca '/etc/openvpn/ns_roadwarrior1/pki/ca.crt' + option cert '/etc/openvpn/ns_roadwarrior1/pki/issued/server.crt' + option crl_verify '/etc/openvpn/ns_roadwarrior1/pki/crl.pem' + option key '/etc/openvpn/ns_roadwarrior1/pki/private/server.key' + option management '/var/run/openvpn_ns_roadwarrior1.socket unix' + option client_to_client '0' + option auth 'SHA256' + option cipher 'AES-256-GCM' + option tls_version_min '1.2' + option ns_auth_mode 'username_password_certificate' + list ns_tag 'automated' + option ns_user_db 'NethService' + option ns_description 'srv1' + option auth_user_pass_verify '/usr/libexec/ns-openvpn/openvpn-remote-auth via-env' + option script_security '3' + list push 'redirect-gateway' + list push 'route 192.168.1.0 255.255.255.0' + list push 'route 172.2.2.0 255.255.254.0' + list ns_public_ip 'vpn.test.org' + +config user 'user1' + option instance 'ns_roadwarrior1' + option ipaddr '10.9.9.50' + option enabled '1' +""" + +ns_plug_db = """ +config main 'config' + option tls_verify '1' + option backup_url 'https://backupd.nethsecurity.org/backup' + option type 'enterprise' + option alerts_url 'https://my.nethsecurity.org/isa/' + option api_url 'https://my.nethsecurity.org/api/' + option inventory_url 'https://my.nethsecurity.org/isa/inventory/store/' + option system_id 'xxx' + option secret 'yyy' + option server '' + option unit_id '' + option token '' +""" + +ban_ip_db = """ +config banip 'global' + option ban_enabled '1' + option ban_debug '1' + option ban_autodetect '0' + list ban_logterm 'Exit before auth from' + option ban_fetchcmd 'curl' + option ban_protov4 '1' + list ban_ifv4 'FIBER100M' + list ban_ifv4 'FTTC_BCK' + list ban_dev 'eth0' + list ban_dev 'eth5' + list ban_allowurl '' + option ban_loginput '1' + option ban_logforwardwan '1' + option ban_logforwardlan '1' + option ban_autoallowlist '1' + option ban_autoblocklist '1' + option ban_allowlistonly '0' + option ban_nftexpiry '1d' + option ban_logcount '3' + list ban_feed 'nethesislvl3' + list ban_feed 'test1' +""" + +ns_ui_db = """ +config main 'config' + option luci_enable '0' + option nsui_enable '1' + option nsui_extra_port '9090' + option nsui_extra_enable '1' +""" + +fstab_db = """ +config global + option anon_swap '0' + option anon_mount '0' + option auto_swap '1' + option auto_mount '1' + option delay_root '5' + option check_fs '0' + +config mount + option target '/rom' + option uuid '520aae54-7d368ea4-c0a96e03-92927a0f' + option enabled '0' + +config mount 'ns_data' + option target '/mnt/data' + option uuid '523ba406-b2b6-4a3c-b071-ac1bf6acf4c6' + option enabled '1' +""" + +nginx_db = """ +config main 'global' + option uci_enable 'true' + +config server '_lan' + list listen '443 ssl default_server' + list listen '[::]:443 ssl default_server' + option server_name '_lan' + list include 'conf.d/*.locations' + option uci_manage_ssl 'custom' + option ssl_certificate '/etc/ssl/acme/test.nethsecurity.org.fullchain.crt' + option ssl_certificate_key '/etc/ssl/acme/test.nethsecurity.org.key' + option ssl_session_cache 'shared:SSL:32k' + option ssl_session_timeout '64m' + option error_log 'syslog:server=unix:/dev/log' + option access_log 'syslog:server=unix:/dev/log' + +config location 'ns_server2_location1' + option uci_server 'ns_server2' + option location '/' + option proxy_pass 'https://172.4.5.4' + +config server 'ns_server1' + option uci_description 'Test 1' + option ssl_session_timeout '64m' + list proxy_set_header 'Host $http_host' + list listen '443 ssl' + list listen '[::]:443 ssl' + option ssl_session_cache 'shared:SSL:32k' + option server_name 'metrics.nethsecurity.org' + list include 'conf.d/ns_server1.proxy' + option uci_manage_ssl 'acme' + option ssl_certificate '/etc/ssl/acme/test.nethsecurity.org.fullchain.crt' + option ssl_certificate_key '/etc/ssl/acme/test.nethsecurity.org.key' + +config location 'ns_server1_location1' + option uci_server 'ns_server1' + option location '/' + option proxy_pass 'https://172.3.3.3' +""" + +ipsec_db = """ +config ipsec 'ns_ipsec_global' + option debug '1' + option zone 'ipsec' + +config crypto_proposal 'ns_6fd94f07_ike' + option encryption_algorithm 'aes256' + option hash_algorithm 'sha256' + option dh_group 'modp2048' + option ns_link 'ipsec/ns_6fd94f07' + +config crypto_proposal 'ns_6fd94f07_esp' + option encryption_algorithm 'aes256' + option hash_algorithm 'sha256' + option dh_group 'modp2048' + option ns_link 'ipsec/ns_6fd94f07' + +config tunnel 'ns_6fd94f07_tunnel_1' + option ipcomp 'false' + option dpdaction 'none' + list local_subnet '192.168.1.0/24' + list remote_subnet '192.168.3.0/24' + option rekeytime '3600' + list crypto_proposal 'ns_6fd94f07_esp' + option closeaction 'none' + option startaction 'start' + option if_id '1' + option ns_link 'ipsec/ns_6fd94f07' + +config tunnel 'ns_6fd94f07_tunnel_2' + option ipcomp 'false' + option dpdaction 'none' + list local_subnet '172.4.4.0/24' + list remote_subnet '192.168.3.0/24' + option rekeytime '3600' + list crypto_proposal 'ns_6fd94f07_esp' + option closeaction 'none' + option startaction 'start' + option if_id '1' + option ns_link 'ipsec/ns_6fd94f07' + +config remote 'ns_6fd94f07' + option ns_name 't1' + option authentication_method 'psk' + option gateway '1.2.3.4' + option keyexchange 'ikev1' + option local_identifier '@l1' + option local_ip '5.6.7.8' + option enabled '1' + option remote_identifier '@r1' + option pre_shared_key 'xxx' + list crypto_proposal 'ns_6fd94f07_ike' + option rekeytime '3600' + list tunnel 'ns_6fd94f07_tunnel_1' + list tunnel 'ns_6fd94f07_tunnel_2' +""" + +dpi_db = """ +config main 'config' + option log_blocked '0' + option enabled '1' + option firewall_exemption '0' + +config rule 'ns_2b170d05' + option enabled '1' + option device 'br-lan' + option action 'block' + list application 'netify.facebook' +""" + +dhcp_db = """ +config dnsmasq + option domainneeded '1' + option boguspriv '1' + option filterwin2k '0' + option localise_queries '1' + option rebind_protection '1' + option rebind_localhost '1' + option local '/lan/' + option domain 'nethesis.it' + option expandhosts '1' + option nonegcache '0' + option cachesize '1000' + option authoritative '1' + option readethers '1' + option leasefile '/tmp/dhcp.leases' + option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto' + option nonwildcard '1' + option localservice '1' + option ednspacket_max '1232' + option filter_aaaa '0' + option filter_a '0' + option logqueries '0' + list server '8.8.8.8' + list server '8.8.4.4' + list rebind_domain 'nethsecurity.org' + +config dhcp 'lan' + option interface 'lan' + option start '100' + option limit '150' + option leasetime '12h' + option dhcpv4 'server' + option dhcpv6 'server' + option ra 'server' + option ra_slaac '1' + list ra_flags 'managed-config' + list ra_flags 'other-config' +""" + +mwan3_db = """ +config globals 'globals' + option mmx_mask '0x3F00' + +config policy 'ns_default' + option label 'Default' + list use_member 'ns_FIBER100M_M10_W100' + list use_member 'ns_FTTC_BCK_M20_W100' + +config interface 'FIBER100M' + option enabled '1' + option initial_state 'online' + option family 'ipv4' + list track_ip '8.8.8.8' + list track_ip '208.67.222.222' + option track_method 'ping' + option reliability '1' + option count '1' + option size '56' + option max_ttl '60' + option timeout '4' + option interval '10' + option failure_interval '5' + option recovery_interval '5' + option down '5' + option up '5' + +config member 'ns_FIBER100M_M10_W100' + option interface 'FIBER100M' + option metric '10' + option weight '100' + +config interface 'FTTC_BCK' + option enabled '1' + option initial_state 'online' + option family 'ipv4' + list track_ip '8.8.8.8' + list track_ip '208.67.222.222' + option track_method 'ping' + option reliability '1' + option count '1' + option size '56' + option max_ttl '60' + option timeout '4' + option interval '10' + option failure_interval '5' + option recovery_interval '5' + option down '5' + option up '5' + +config member 'ns_FTTC_BCK_M20_W100' + option interface 'FTTC_BCK' + option metric '20' + option weight '100' + +config rule 'ns_default_rule' + option label 'Default Rule' + option use_policy 'ns_default' + option sticky '1' +""" + +qosify_db = """ +config interface 'wan' + option name 'wan' + option disabled '1' + option bandwidth_up '100mbit' + option bandwidth_down '100mbit' + option overhead_type 'none' + option ingress '1' + option egress '1' + option mode 'diffserv4' + option nat '1' + option host_isolate '1' + option autorate_ingress '0' + +config device 'wandev' + option disabled '1' + option name 'wan' + option bandwidth '100mbit' + +config interface 'FIBER100M' + option name 'FIBER100M' + option disabled '0' + option bandwidth_up '85mbit' + option bandwidth_down '85mbit' + +config interface 'FTTC_BCK' + option name 'FTTC_BCK' + option disabled '0' + option bandwidth_up '20mbit' + option bandwidth_down '40mbit' +""" + +objects_db = """ + +config rule 'ns_b606fef5' + option name 'object1' + option dest '8.8.8.0/24' + option map_from '12.0.0.0/24' + option map_to '16.0.0.0/24' + +config rule 'ns_003ff675' + option name 'object2' + option src '12.0.0.0/24' + option map_from '15.0.0.0/24' + option map_to '14.0.0.0/24' + +config domain 'ns_7c9c0852' + option name 'object3' + option family 'ipv4' + option timeout '660' + list domain 'foo.com' + +config host 'ns_bcfb447b' + option name 'object4' + option family 'ipv4' + list ipaddr 'dhcp/ns_8d5d2cf4' +""" + +adblock_db = """ +config adblock 'global' + option adb_enabled '1' + option adb_debug '0' + option adb_forcedns '1' + option adb_safesearch '0' + option adb_dnsfilereset '0' + option adb_mail '0' + option adb_report '0' + option adb_backup '1' + option adb_fetchutil 'wget' + option adb_dns 'dnsmasq' + option ts_enabled '1' + list adb_zonelist 'lan' + list adb_portlist '53' + list adb_portlist '853' + option adb_srcarc '/etc/adblock/combined.sources.gz' + option adb_dnsinstance '0' + option adb_fetchparm '--compression=gzip --no-cache --no-cookies --max-redirect=0 --timeout=20 -O' + list adb_sources 'adaway' + list adb_sources 'adguard' + list adb_sources 'disconnect' + list adb_sources 'yoyo' + list adb_sources 'malware_lvl2' + list adb_sources 'yoroi_susp_level2' + list adb_sources 'yoroi_malware_level1' +""" + +rpcd_db = """ +config rpcd + option dummy 'to get ignored' + +config login + option username 'root' + +config login 'controller' + option username 'gibblerish' + +config login 'ns_7fe3e016' + option username 'example' +""" + +def _setup_db(tmp_path): + # setup fake db + with tmp_path.joinpath('network').open('w') as fp: + fp.write(network_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('dedalo').open('w') as fp: + fp.write(dedalo_db) + with tmp_path.joinpath('flashstart').open('w') as fp: + fp.write(flashstart_db) + with tmp_path.joinpath('openvpn').open('w') as fp: + fp.write(openvpn_db) + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write(ns_plug_db) + with tmp_path.joinpath('banip').open('w') as fp: + fp.write(ban_ip_db) + with tmp_path.joinpath('ns-ui').open('w') as fp: + fp.write(ns_ui_db) + with tmp_path.joinpath('fstab').open('w') as fp: + fp.write(fstab_db) + with tmp_path.joinpath('nginx').open('w') as fp: + fp.write(nginx_db) + with tmp_path.joinpath('ipsec').open('w') as fp: + fp.write(ipsec_db) + with tmp_path.joinpath('dpi').open('w') as fp: + fp.write(dpi_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('mwan3').open('w') as fp: + fp.write(mwan3_db) + with tmp_path.joinpath('qosify').open('w') as fp: + fp.write(qosify_db) + with tmp_path.joinpath('netmap').open('w') as fp: + fp.write(netmap_db) + with tmp_path.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with tmp_path.joinpath('adblock').open('w') as fp: + fp.write(adblock_db) + with tmp_path.joinpath('rpcd').open('w') as fp: + fp.write(rpcd_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_fact_hotspot(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_hotspot(u) == {"enabled": True, "server": "https://my.nethspot.com/api", "interface": "eth2.10"} + +def test_fact_flashstart(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_flashstart(u) == {"enabled": False, "bypass": 2, "custom_servers": 1, "pro_plus": True} + +def test_fact_openvpn_rw(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_openvpn_rw(u) == { + "enabled": 1, + "server": 1, + "instances": [ + { + "section": "ns_roadwarrior1", + "authentication": "username_password_certificate", + "user_database": "NethService", + "mode": "tun" + } + ] + } + +def test_fact_openvpn_tun(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_openvpn_tun(u) + assert result == { + "client": 0, + "server": 1, + "tunnels": [ + { + "section": "ns_openvpn1", + "mode": "tun" + } + ] + } + +def test_fact_subscription_status(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_subscription_status(u) == {"status": "enterprise"} + +def test_fact_controller(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_controller(u) == {"enabled": False} + +def test_fact_rpcd_users(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_rpcd_users(u) == {"count": 1} + +def test_fact_threat_shield(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_threat_shield(u) == {"enabled": True, "community": 1, "enterprise": 1} + +def test_fact_ui(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_ui(u) == {"luci": False, "port443": True, "port9090": True} + +def test_fact_storage(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_storage(u) == {"enabled": True} + +def test_fact_network(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_network(u) + assert result['zones'][0]['name'] == 'lan' + assert result['zones'][0]['ipv4'] == 1 + assert result['zones'][0]['ipv6'] == 0 + assert result['zones'][1]['name'] == 'grey' + assert result['zones'][1]['ipv4'] == 1 + assert result['zones'][1]['ipv6'] == 0 + assert result['zones'][2]['name'] == 'orange' + assert result['zones'][2]['ipv4'] == 0 + assert result['zones'][2]['ipv6'] == 0 + assert result['zones'][3]['name'] == 'wan' + assert result['zones'][3]['ipv4'] == 1 + assert result['zones'][3]['ipv6'] == 0 + assert result['zones'][4]['name'] == 'mytrusted' + assert result['zones'][4]['ipv4'] == 0 + assert result['zones'][4]['ipv6'] == 0 + assert result['zones'][5]['name'] == 'mylinked' + assert result['zones'][5]['ipv4'] == 0 + assert result['zones'][5]['ipv6'] == 0 + assert result['zones'][6]['name'] == 'mytrusted2' + assert result['zones'][6]['ipv4'] == 0 + assert result['zones'][6]['ipv6'] == 0 + assert result['zones'][7]['name'] == 'blue' + assert result['zones'][7]['ipv4'] == 1 + assert result['zones'][7]['ipv6'] == 0 + assert result['interface_counts']['vlans'] == 0 + assert result['interface_counts']['bridges'] == 2 + assert result['interface_counts']['bonds'] == 0 + assert result['zone_network_counts']['lan'] == 2 + assert result['zone_network_counts']['grey'] == 1 + assert result['zone_network_counts']['orange'] == 0 + assert result['zone_network_counts']['wan'] == 2 + assert result['zone_network_counts']['mytrusted'] == 0 + assert result['zone_network_counts']['mylinked'] == 0 + assert result['zone_network_counts']['mytrusted2'] == 0 + assert result['zone_network_counts']['blue'] == 1 + assert result['route_info']['count_ipv6_route'] == 1 + assert result['route_info']['count_ipv4_route'] == 1 + +def test_fact_proxy_pass(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_proxy_pass(u) == {"count": 2} + +def test_fact_ipsec(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_ipsec(u) == {"count": 1} + +def test_fact_dpi(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_dpi(u) == {"enabled": True, "rules": 1} + +def test_fact_dhcp_server(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_dhcp_server(u) == {"count": 1, "static_leases": 0, "dynamic_leases": 0, "dns_records_count": 0, "dns_forwarder_enabled": True } + +def test_fact_multiwan(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_multiwan(u) + assert result['enabled'] + assert result['policies']['backup'] == 1 + assert result['policies']['balance'] == 0 + assert result['policies']['custom'] == 0 + assert result['rules'] == 1 + +def test_fact_qos(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_qos(u) + assert result['count'] == 2 + assert result['rules'][0]['enabled'] == True + assert result['rules'][0]['upload'] == 85 + assert result['rules'][0]['download'] == 85 + assert result['rules'][1]['enabled'] == True + assert result['rules'][1]['upload'] == 20 + assert result['rules'][1]['download'] == 40 + +def test_fact_certificates_info(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_certificates_info(u) + assert result['custom_certificates']['count'] == 0 + assert result['acme_certificates']['count'] == 0 + assert result['acme_certificates']['issued'] == 0 + assert result['acme_certificates']['pending'] == 0 + +def test_fact_firewall_stats(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_firewall_stats(u) + + # Validate the 'firewall' section + assert result['firewall']['port_forward'] == 2 + assert result['firewall']['nat']['masquerade'] == 1 + assert result['firewall']['nat']['snat'] == 4 + assert result['firewall']['nat']['accept'] == 2 + assert result['firewall']['netmap']['source'] == 3 + assert result['firewall']['netmap']['destination'] == 2 + assert result['firewall']['rules']['forward'] == 19 + assert result['firewall']['rules']['input'] == 7 + assert result['firewall']['rules']['output'] == 2 + + # Validate the 'objects' section + assert result['objects']['domains'] == 3 + assert result['objects']['hosts'] == 3 + assert result['objects']['port_forward']['allowed_from'] == 2 + assert result['objects']['port_forward']['destination_to'] == 2 + assert result['objects']['mwan_rules'] == 0 + assert result['objects']['rules']['forward'] == 1 + assert result['objects']['rules']['input'] == 1 + assert result['objects']['rules']['output'] == 1 + +def test_fact_adblock(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_adblock(u) + assert result == {"enabled": True, "community": 5, "enterprise": 2} + +# Tests for info_* helper functions + +def test_info_kernel_version(): + """Test info_kernel_version reads from /proc/version""" + u = EUci() + + # Test successful read + mock_version = "Linux version 5.10.176 (builder@nethsec) (gcc version 11.2.0) #0 SMP Mon Jan 1 00:00:00 2024\n" + with patch('builtins.open', mock_open(read_data=mock_version)): + result = inventory.info_kernel_version(u) + assert result == "5.10.176" + + # Test exception handling (file not found) + with patch('builtins.open', side_effect=FileNotFoundError): + result = inventory.info_kernel_version(u) + assert result == '' + +def test_info_uptime_seconds(): + """Test info_uptime_seconds reads from /proc/uptime""" + u = EUci() + + # Test successful read + mock_uptime = "12345.67 98765.43\n" + with patch('builtins.open', mock_open(read_data=mock_uptime)): + result = inventory.info_uptime_seconds(u) + assert result == 12345 + + # Test exception handling + with patch('builtins.open', side_effect=FileNotFoundError): + result = inventory.info_uptime_seconds(u) + assert result == 0 + +def test_info_fqdn(tmp_path): + """Test info_fqdn retrieves hostname from UCI""" + # Setup system config + system_db = """ +config system + option hostname 'myhost' + option domain 'example.com' +""" + with tmp_path.joinpath('system').open('w') as fp: + fp.write(system_db) + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + + u = EUci(confdir=tmp_path.as_posix()) + result = inventory.info_fqdn(u) + # Should be anonymized since no subscription + assert result.startswith('anon-') + +def test_info_default_ipv4_dig_method(tmp_path): + """Test info_default_ipv4 with dig method""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock successful dig command + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '"192.0.2.1"\n' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_default_ipv4(u) + assert result.startswith('anon-') # Should be anonymized + +def test_info_default_ipv4_curl_fallback(tmp_path): + """Test info_default_ipv4 falls back to curl when dig fails""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock dig failure and curl success + def mock_subprocess_run(cmd, **kwargs): + result = MagicMock() + if 'dig' in cmd: + result.returncode = 1 + result.stdout = '' + elif 'curl' in cmd: + result.returncode = 0 + result.stdout = '203.0.113.1\n' + else: + result.returncode = 1 + result.stdout = '' + return result + + with patch('subprocess.run', side_effect=mock_subprocess_run): + result = inventory.info_default_ipv4(u) + assert result.startswith('anon-') + +def test_info_default_ipv4_ip_command_fallback(tmp_path): + """Test info_default_ipv4 falls back to ip command when dig and curl fail""" + with tmp_path.joinpath('network').open('w') as fp: + fp.write(network_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock dig and curl failure, ip command success + def mock_subprocess_run(cmd, **kwargs): + result = MagicMock() + if isinstance(cmd, list) and 'ip' in cmd: + result.returncode = 0 + result.stdout = '[{"addr_info": [{"family": "inet", "local": "198.51.100.1"}]}]' + else: + result.returncode = 1 + result.stdout = '' + return result + + with patch('subprocess.run', side_effect=mock_subprocess_run): + with patch('nethsec.utils.get_all_wan_devices', return_value=['eth0']): + result = inventory.info_default_ipv4(u) + assert result.startswith('anon-') + +def test_info_default_ipv4_all_methods_fail(tmp_path): + """Test info_default_ipv4 returns empty string when all methods fail""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock all methods failing + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stdout = '' + + with patch('subprocess.run', return_value=mock_result): + with patch('nethsec.utils.get_all_wan_devices', return_value=[]): + result = inventory.info_default_ipv4(u) + assert result == '' + +def test_info_default_ipv6_no_wan_ipv6(tmp_path): + """Test info_default_ipv6 returns empty when no IPv6 on WAN""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock no WAN devices or no IPv6 + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '[]' + + with patch('subprocess.run', return_value=mock_result): + with patch('nethsec.utils.get_all_wan_devices', return_value=['eth0']): + result = inventory.info_default_ipv6(u) + assert result == '' + +def test_info_default_ipv6_with_wan_ipv6(tmp_path): + """Test info_default_ipv6 with IPv6 on WAN""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock WAN with IPv6 + def mock_subprocess_run(cmd, **kwargs): + result = MagicMock() + if isinstance(cmd, list) and 'ip' in cmd and '-6' in cmd: + result.returncode = 0 + result.stdout = '[{"addr_info": [{"family": "inet6", "local": "2001:db8::1", "scope": "global"}]}]' + elif isinstance(cmd, list) and 'dig' in cmd: + result.returncode = 0 + result.stdout = '"2001:db8::2"\n' + else: + result.returncode = 1 + result.stdout = '' + return result + + with patch('subprocess.run', side_effect=mock_subprocess_run): + with patch('nethsec.utils.get_all_wan_devices', return_value=['eth0']): + result = inventory.info_default_ipv6(u) + assert result.startswith('anon-') + +def test_info_default_ipv6_fallback_to_curl(tmp_path): + """Test info_default_ipv6 falls back to curl when dig fails""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock WAN with IPv6, dig fails, curl succeeds + def mock_subprocess_run(cmd, **kwargs): + result = MagicMock() + if isinstance(cmd, list) and 'ip' in cmd and '-6' in cmd: + result.returncode = 0 + result.stdout = '[{"addr_info": [{"family": "inet6", "local": "2001:db8::1", "scope": "global"}]}]' + elif isinstance(cmd, list) and 'curl' in cmd: + result.returncode = 0 + result.stdout = '2001:db8::3\n' + else: + result.returncode = 1 + result.stdout = '' + return result + + with patch('subprocess.run', side_effect=mock_subprocess_run): + with patch('nethsec.utils.get_all_wan_devices', return_value=['eth0']): + result = inventory.info_default_ipv6(u) + assert result.startswith('anon-') + +def test_info_package_updates_available_with_updates(): + """Test info_package_updates_available when updates are available""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"updates": ["package1", "package2"]}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_package_updates_available(u) + assert result is True + +def test_info_package_updates_available_no_updates(): + """Test info_package_updates_available when no updates available""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"updates": []}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_package_updates_available(u) + assert result is False + +def test_info_package_updates_available_command_fails(): + """Test info_package_updates_available when command fails""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stdout = '' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_package_updates_available(u) + assert result is False + +def test_parse_version_with_prefix(): + """Test parse_version removes NethSecurity prefix""" + assert inventory.parse_version('NethSecurity 8.1.0-dev') == (8, 1, 0) + assert inventory.parse_version('NethSecurity 8.2.1') == (8, 2, 1) + +def test_parse_version_without_prefix(): + """Test parse_version handles versions without prefix""" + assert inventory.parse_version('8.1.0-dev') == (8, 1, 0) + assert inventory.parse_version('8.2.1') == (8, 2, 1) + +def test_parse_version_comparison(): + """Test parse_version version comparison logic""" + v1 = inventory.parse_version('8.1.0') + v2 = inventory.parse_version('8.2.0') + v3 = inventory.parse_version('8.1.1') + + assert v2 > v1 + assert v3 > v1 + assert v2 > v3 + +def test_parse_version_invalid(): + """Test parse_version handles invalid input""" + result = inventory.parse_version('invalid') + assert result == () + + result = inventory.parse_version('') + assert result == () + +def test_info_image_updates_available_update_exists(): + """Test info_image_updates_available when newer version exists""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.1.0", "lastVersion": "NethSecurity 8.2.0"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is True + +def test_info_image_updates_available_no_update(): + """Test info_image_updates_available when current version is latest""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.2.0", "lastVersion": "NethSecurity 8.2.0"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is False + +def test_info_image_updates_available_patch_update(): + """Test info_image_updates_available when a patch update is available""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.2.0", "lastVersion": "NethSecurity 8.2.1"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is True + +def test_info_image_updates_available_minor_update(): + """Test info_image_updates_available when a minor update is available""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.2.0", "lastVersion": "NethSecurity 8.3.0"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is True + +def test_info_image_updates_available_current_newer(): + """Test info_image_updates_available when current version is newer than last""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.3.0", "lastVersion": "NethSecurity 8.2.0"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is False + +def test_info_image_updates_available_command_fails(): + """Test info_image_updates_available when command fails""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stdout = '' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is False diff --git a/packages/python3-nethsec/tests/test_ipsec.py b/packages/python3-nethsec/tests/test_ipsec.py new file mode 100644 index 000000000..ba1b38ef4 --- /dev/null +++ b/packages/python3-nethsec/tests/test_ipsec.py @@ -0,0 +1,89 @@ +import pathlib + +import pytest +from euci import EUci +from pytest_mock import MockFixture +from nethsec import ipsec, utils + +templates_db = """ +config template_rule 'ns_ipsec_esp' + option name 'Allow-IPSec-ESP' + option src 'wan' + option proto 'esp' + option target 'ACCEPT' + +config template_rule 'ns_ipsec_ike' + option name 'Allow-IPSec-IKE' + option src 'wan' + option dest_port '500' + option proto 'udp' + option target 'ACCEPT' + +config template_rule 'ns_ipsec_nat' + option name 'Allow-IPSec-NAT' + option src 'wan' + option dest_port '500' + option proto 'udp' + option target 'ACCEPT' +""" + +@pytest.fixture +def e_uci(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + return EUci(confdir=conf_dir.as_posix(), savedir=save_dir.as_posix()) + + +@pytest.fixture +def e_uci_with_data(e_uci: EUci): + with pathlib.Path(e_uci.confdir()).joinpath('templates').open('w') as fp: + fp.write(templates_db) + with pathlib.Path(e_uci.confdir()).joinpath('firewall').open('a') as fp: + pass + return e_uci + +def test_init_ipsec(e_uci): + ipsec.init_ipsec(e_uci) + assert(e_uci.get('ipsec', 'ns_ipsec_global') == 'ipsec') + assert(e_uci.get('ipsec', 'ns_ipsec_global', 'debug') == '0') + assert(e_uci.get('ipsec', 'ns_ipsec_global', 'zone') == ipsec.IPSEC_ZONE) + +def test_open_firewall_ports(e_uci_with_data): + ipsec.open_firewall_ports(e_uci_with_data) + nat = ike = esp = False + for r in utils.get_all_by_type(e_uci_with_data, 'firewall', 'rule'): + name = e_uci_with_data.get('firewall', r, 'name') + if name == 'Allow-IPSec-NAT': + nat = True + elif name == 'Allow-IPSec-IKE': + ike = True + elif name == 'Allow-IPSec-ESP': + esp = True + assert (nat and ipsec and esp) + +def test_add_trusted_interface(e_uci_with_data): + ipsec.add_trusted_interface(e_uci_with_data, 'ipsec1') + count = 0 + zid = '' + # check the zone has been created + for section in e_uci_with_data.get_all('firewall'): + if e_uci_with_data.get('firewall', section) == 'zone': + if e_uci_with_data.get('firewall', section, 'name') == ipsec.IPSEC_ZONE: + count = count + 1 + zid = section + assert(count == 1) + assert(zid) + assert(e_uci_with_data.get_all('firewall', zid, 'network') == ('ipsec1',)) + # check the zone has not been duplicated + count = 0 + ipsec.add_trusted_interface(e_uci_with_data, 'ipsec2') + for section in e_uci_with_data.get_all('firewall'): + if e_uci_with_data.get('firewall', section) == 'zone': + if e_uci_with_data.get('firewall', section, 'name') == ipsec.IPSEC_ZONE: + count = count + 1 + assert(count == 1) + assert('ipsec1' in e_uci_with_data.get_all('firewall', zid, 'network')) + assert('ipsec2' in e_uci_with_data.get_all('firewall', zid, 'network')) + diff --git a/packages/python3-nethsec/tests/test_mwan.py b/packages/python3-nethsec/tests/test_mwan.py new file mode 100644 index 000000000..5b6a9ce33 --- /dev/null +++ b/packages/python3-nethsec/tests/test_mwan.py @@ -0,0 +1,610 @@ +import pathlib + +import pytest +from euci import EUci + +from nethsec import mwan, objects +from nethsec.utils import ValidationError + +network_db = """ +config interface 'loopback' + option device 'lo' + option proto 'static' + option ipaddr '127.0.0.1' + option netmask '255.0.0.0' + +config interface 'GREEN_1' + option proto 'static' + option device 'eth0' + option ipaddr '192.168.200.2' + option netmask '255.255.255.0' + +config interface 'RED_1' + option proto 'static' + option device 'eth1' + option ipaddr '10.0.0.2' + option netmask '255.255.255.0' + option gateway '10.0.0.1' + +config interface 'RED_2' + option proto 'static' + option device 'eth2' + option ipaddr '10.0.1.2' + option netmask '255.255.255.0' + option gateway '10.0.1.1' + +config interface 'RED_3' + option proto 'static' + option device 'eth3' + option ipaddr '10.0.2.2' + option netmask '255.255.255.0' + option gateway '10.0.2.1' + +config device + option name 'eth0' + +config device + option name 'eth1' + +config device + option name 'eth2' + +config device + option name 'eth3' +""" + +ns_api_db = """ +config defaults_mwan 'defaults_mwan' + option initial_state 'online' + option protocol 'ipv4' + list track_ip '8.8.8.8' + list track_ip '208.67.222.222' + option tracking_method 'ping' + option tracking_reliability '1' + option ping_count '1' + option ping_size '56' + option ping_max_ttl '60' + option ping_timeout '4' + option ping_interval '10' + option ping_failure_interval '5' + option ping_recovery_interval '5' + option interface_down_threshold '5' + option interface_up_threshold '5' + option link_quality '0' + option quality_failure_latency '1000' + option quality_failure_packet_loss '40' + option quality_recovery_latency '500' + option quality_recovery_packet_loss '10' +""" + +objects_db = """ +""" + +dhcp_db = """ +config domain 'ns_domain_mwan' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' + +config host 'ns_host_mwan' + option ip '192.168.100.5' + option mac 'fe:54:00:6a:50:bf' + option dns '1' + option name 'host2' + option ns_description 'host2' +""" + +users_db = """ +""" + +firewall_db = """ +""" + +@pytest.fixture +def e_uci(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + with conf_dir.joinpath('network').open('w') as fp: + fp.write(network_db) + with conf_dir.joinpath('ns-api').open('w') as fp: + fp.write(ns_api_db) + with conf_dir.joinpath('mwan3').open('w') as fp: + fp.write('') + with conf_dir.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with conf_dir.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with conf_dir.joinpath('users').open('w') as fp: + fp.write(users_db) + with conf_dir.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + return EUci(confdir=conf_dir.as_posix(), savedir=save_dir.as_posix()) + + +def test_create_interface(e_uci): + assert mwan.__store_interface(e_uci, 'RED_1') == (True, True) + assert mwan.__store_interface(e_uci, 'RED_2') == (True, True) + # assert every interface has defaults + assert e_uci.get('mwan3', 'RED_1') == 'interface' + assert e_uci.get('mwan3', 'RED_1', 'enabled') == '1' + assert e_uci.get('mwan3', 'RED_1', 'initial_state') == 'online' + assert e_uci.get('mwan3', 'RED_1', 'family') == 'ipv4' + assert e_uci.get('mwan3', 'RED_1', 'track_ip', list=True) == ('8.8.8.8', '208.67.222.222') + assert e_uci.get('mwan3', 'RED_1', 'track_method') == 'ping' + assert e_uci.get('mwan3', 'RED_1', 'reliability') == '1' + assert e_uci.get('mwan3', 'RED_1', 'count') == '1' + assert e_uci.get('mwan3', 'RED_1', 'size') == '56' + assert e_uci.get('mwan3', 'RED_1', 'max_ttl') == '60' + assert e_uci.get('mwan3', 'RED_1', 'timeout') == '4' + assert e_uci.get('mwan3', 'RED_1', 'interval') == '10' + assert e_uci.get('mwan3', 'RED_1', 'failure_interval') == '5' + assert e_uci.get('mwan3', 'RED_1', 'recovery_interval') == '5' + assert e_uci.get('mwan3', 'RED_1', 'down') == '5' + assert e_uci.get('mwan3', 'RED_1', 'up') == '5' + # assert interface has metric + assert e_uci.get('network', 'RED_1', 'metric') == '30' + assert e_uci.get('network', 'RED_2', 'metric') == '40' + assert mwan.__store_interface(e_uci, 'RED_1') == (False, False) + + +def test_fail_create_invalid_interface(e_uci): + with pytest.raises(ValueError) as err: + mwan.__store_interface(e_uci, 'RED_4') + assert err.value.args[0] == 'name' + assert err.value.args[1] == 'invalid' + assert err.value.args[2] == 'RED_4' + + +def test_interface_avoid_edit_of_metric(e_uci): + e_uci.set('network', 'RED_1', 'metric', '10') + assert mwan.__store_interface(e_uci, 'RED_1') == (True, False) + + +def test_create_member(e_uci): + assert mwan.__store_member(e_uci, 'RED_1', 10, 100) == ('ns_RED_1_M10_W100', True) + assert mwan.__store_member(e_uci, 'RED_1', 10, 100) == ('ns_RED_1_M10_W100', False) + assert mwan.__store_member(e_uci, 'RED_1', 1, 100) == ('ns_RED_1_M1_W100', True) + + +def test_create_default_mwan(e_uci, mocker): + mocker.patch('subprocess.run') + assert mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '20', + 'weight': '100', + } + ]) == ['mwan3.ns_default', + 'mwan3.RED_1', + 'network.RED_1', + 'mwan3.ns_RED_1_M10_W200', + 'mwan3.RED_2', + 'network.RED_2', + 'mwan3.ns_RED_2_M20_W100', + 'mwan3.ns_default_rule'] + + assert e_uci.get('mwan3', 'ns_default') == 'policy' + assert e_uci.get('mwan3', 'ns_default', 'label') == 'default' + assert e_uci.get('mwan3', 'ns_default', 'use_member', list=True) == ( + 'ns_RED_1_M10_W200', 'ns_RED_2_M20_W100') + assert e_uci.get('mwan3', 'ns_default', 'last_resort') == 'default' + + +def test_create_unique_mwan(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'this', []) + with pytest.raises(ValueError): + mwan.store_policy(e_uci, 'this', []) + + +def test_metric_generation(e_uci): + assert mwan.__generate_metric(e_uci) == 30 + assert mwan.__store_interface(e_uci, 'RED_1') == (True, True) + assert mwan.__generate_metric(e_uci) == 40 + assert mwan.__generate_metric(e_uci) == 40 + assert mwan.__store_interface(e_uci, 'RED_2') == (True, True) + assert mwan.__generate_metric(e_uci) == 50 + + +def test_list_policies(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'backup', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '20', + 'weight': '100', + } + ]) + mwan.store_policy(e_uci, 'balance', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + mwan.store_policy(e_uci, 'custom', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + index = mwan.index_policies(e_uci) + # check backup policy + assert index[0]['name'] == 'ns_backup' + assert index[0]['label'] == 'backup' + assert index[0]['type'] == 'backup' + assert index[0]['members'][10][0]['name'] == 'ns_RED_1_M10_W200' + assert index[0]['members'][10][0]['interface'] == 'RED_1' + assert index[0]['members'][10][0]['metric'] == '10' + assert index[0]['members'][10][0]['weight'] == '200' + assert index[0]['members'][20][0]['name'] == 'ns_RED_2_M20_W100' + assert index[0]['members'][20][0]['interface'] == 'RED_2' + assert index[0]['members'][20][0]['metric'] == '20' + assert index[0]['members'][20][0]['weight'] == '100' + # check balance policy + assert index[1]['name'] == 'ns_balance' + assert index[1]['label'] == 'balance' + assert index[1]['type'] == 'balance' + assert index[1]['members'][10][0]['name'] == 'ns_RED_3_M10_W200' + assert index[1]['members'][10][0]['interface'] == 'RED_3' + assert index[1]['members'][10][0]['metric'] == '10' + assert index[1]['members'][10][0]['weight'] == '200' + assert index[1]['members'][10][1]['name'] == 'ns_RED_2_M10_W100' + assert index[1]['members'][10][1]['interface'] == 'RED_2' + assert index[1]['members'][10][1]['metric'] == '10' + assert index[1]['members'][10][1]['weight'] == '100' + # check custom policy + assert index[2]['name'] == 'ns_custom' + assert index[2]['label'] == 'custom' + assert index[2]['type'] == 'custom' + assert index[2]['members'][10][0]['name'] == 'ns_RED_3_M10_W200' + assert index[2]['members'][10][0]['interface'] == 'RED_3' + assert index[2]['members'][10][0]['metric'] == '10' + assert index[2]['members'][10][0]['weight'] == '200' + assert index[2]['members'][10][1]['name'] == 'ns_RED_2_M10_W100' + assert index[2]['members'][10][1]['interface'] == 'RED_2' + assert index[2]['members'][10][1]['metric'] == '10' + assert index[2]['members'][10][1]['weight'] == '100' + assert index[2]['members'][20][0]['name'] == 'ns_RED_1_M20_W100' + assert index[2]['members'][20][0]['interface'] == 'RED_1' + assert index[2]['members'][20][0]['metric'] == '20' + assert index[2]['members'][20][0]['weight'] == '100' + +def test_policy_length(e_uci, mocker): + mocker.patch('subprocess.run') + with pytest.raises(ValidationError) as e: + assert mwan.store_policy(e_uci, 'nameisa15maxlength', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '20', + 'weight': '100', + } + ]) + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'length_12_max' + +def test_store_rule(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + assert mwan.store_rule(e_uci, 'rule 1', 'ns_default', 'udp', '192.168.1.1/24', '1-1024', '10.0.0.2/12', + '22,443', True) == 'mwan3.ns_rule_1' + assert e_uci.get('mwan3', 'ns_rule_1') == 'rule' + assert e_uci.get('mwan3', 'ns_rule_1', 'label') == 'rule 1' + assert e_uci.get('mwan3', 'ns_rule_1', 'use_policy') == 'ns_default' + assert e_uci.get('mwan3', 'ns_rule_1', 'proto') == 'udp' + assert e_uci.get('mwan3', 'ns_rule_1', 'src_ip') == '192.168.1.1/24' + assert e_uci.get('mwan3', 'ns_rule_1', 'src_port') == '1:1024' + assert e_uci.get('mwan3', 'ns_rule_1', 'dest_ip') == '10.0.0.2/12' + assert e_uci.get('mwan3', 'ns_rule_1', 'dest_port') == '22,443' + assert e_uci.get('mwan3', 'ns_rule_1', 'sticky') == '1' + + domain_id = objects.add_domain_set(e_uci, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + with pytest.raises(ValueError): + mwan.store_rule(e_uci, 'r_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst=f"objects/{domain_id}") + + hostset_id = objects.add_host_set(e_uci, "myhostset", "ipv4", ["192.168.1.1"]) + id = mwan.store_rule(e_uci, 'r_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst=f"objects/{hostset_id}") + + id = id.split('.')[1] + assert e_uci.get('mwan3', id, 'ns_src') == "dhcp/ns_host_mwan" + assert e_uci.get('mwan3', id, 'ns_dst') == f"objects/{hostset_id}" + with pytest.raises(ValueError): + mwan.store_rule(e_uci, 'rule_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst="objects/invalid") + with pytest.raises(ValueError): + mwan.store_rule(e_uci, 'rule_with_obj', 'ns_default', 'udp', ns_src=f"objects/{domain_id}") + +def test_unique_rule(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + with pytest.raises(ValidationError) as e: + mwan.store_rule(e_uci, 'rule 1', 'ns_default') + mwan.store_rule(e_uci, 'rule 1', 'ns_default') + + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'unique' + +def test_rule_length(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + with pytest.raises(ValidationError) as e: + mwan.store_rule(e_uci, 'nameisa15maxlength', 'ns_default') + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'length_12_max' + + +def test_missing_policy_rule(e_uci): + with pytest.raises(ValidationError) as e: + mwan.store_rule(e_uci, 'cool rule', 'ns_default') + + assert e.value.args[0] == 'policy' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'ns_default' + + +def test_delete_non_existent_policy(e_uci, mocker): + mocker.patch('subprocess.run') + with pytest.raises(ValidationError) as e: + mwan.delete_policy(e_uci, 'ns_default') + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'ns_default' + + +def test_delete_policy(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + assert mwan.delete_policy(e_uci, 'ns_default') == ['mwan3.ns_default'] + assert e_uci.get('mwan3', 'ns_default', default=None) is None + + +def test_edit_policy(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + assert mwan.index_policies(e_uci)[0]['type'] == 'balance' + assert mwan.edit_policy(e_uci, 'ns_default', 'new label', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + }, + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '100', + } + ]) == ['mwan3.ns_default', 'mwan3.ns_RED_1_M20_W100', 'mwan3.RED_3', 'network.RED_3', 'mwan3.ns_RED_3_M10_W100'] + assert e_uci.get('mwan3', 'ns_default', 'label') == 'new label' + assert mwan.index_policies(e_uci)[0]['type'] == 'backup' + + +def test_missing_policy(e_uci, mocker): + mocker.patch('subprocess.run') + with pytest.raises(ValidationError) as e: + mwan.edit_policy(e_uci, 'dummy', '', []) + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'dummy' + + +def test_index_rules(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + mwan.store_rule(e_uci, 'rule 1', 'ns_default') + index = mwan.index_rules(e_uci) + assert index[0] == { + 'name': 'ns_default_rule', + 'label': 'Default Rule', + 'policy': { + 'name': 'ns_default', + 'label': 'default', + }, + "sticky": False, + } + assert index[1] == { + 'name': 'ns_rule_1', + 'label': 'rule 1', + 'policy': { + 'name': 'ns_default', + 'label': 'default', + }, + "sticky": False, + } + + +def test_delete_rule(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + mwan.store_rule(e_uci, 'rule 1', 'ns_default') + assert mwan.delete_rule(e_uci, 'ns_rule_1') == 'mwan3.ns_rule_1' + assert 'ns_rule_1' not in e_uci.get_all('mwan3').keys() + + +def test_edit_rule(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'hello world', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + mwan.store_policy(e_uci, 'cool policy', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + } + ]) + assert mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label!', 'udp', '192.168.10.1/12', '80,443', + '0.0.0.0/0', '4040-8080') == 'mwan3.ns_default_rule' + assert e_uci.get('mwan3', 'ns_default_rule', 'label') == 'new label!' + assert e_uci.get('mwan3', 'ns_default_rule', 'use_policy') == 'ns_cool_policy' + assert e_uci.get('mwan3', 'ns_default_rule', 'proto') == 'udp' + assert e_uci.get('mwan3', 'ns_default_rule', 'src_ip') == '192.168.10.1/12' + assert e_uci.get('mwan3', 'ns_default_rule', 'src_port') == '80,443' + assert e_uci.get('mwan3', 'ns_default_rule', 'dest_ip') == '0.0.0.0/0' + assert e_uci.get('mwan3', 'ns_default_rule', 'dest_port') == '4040:8080' + + assert mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label2!', ns_src="dhcp/ns_host_mwan") + assert e_uci.get('mwan3', 'ns_default_rule', 'ns_src') == "dhcp/ns_host_mwan" + + +def test_cant_edit_invalid_rule(e_uci, mocker): + mocker.patch('subprocess.run') + with pytest.raises(ValidationError) as e: + mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label!') + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'ns_default_rule' + mwan.store_policy(e_uci, 'hello world', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + with pytest.raises(ValidationError) as e: + mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label!') + assert e.value.args[0] == 'policy' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'ns_cool_policy' + +def test_policy_type(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'custom', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '200', + } + ]) + assert mwan.index_policies(e_uci)[0]['type'] == 'custom' + +def test_update_rules(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'cool policy', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + } + ]) + domain_id = objects.add_domain_set(e_uci, "mydomainset7", "ipv4", ["test1.com", "test2.com"]) + with pytest.raises(ValidationError): + mwan.store_rule(e_uci, 'r_with_obj', 'ns_cool_policy', 'udp', ns_src="dhcp/ns_domain_mwan", ns_dst=f"objects/{domain_id}") + + id = mwan.store_rule(e_uci, 'r_with_obj', 'ns_cool_policy', 'udp', ns_src="dhcp/ns_domain_mwan", ns_dst=f"dhcp/ns_host_mwan") + id = id.split('.')[1] + mwan.update_rules(e_uci) + assert e_uci.get('mwan3', id, 'src_ip') == '7.8.9.1' + assert e_uci.get('mwan3', id, 'dest_ip') == '192.168.100.5' diff --git a/packages/python3-nethsec/tests/test_objects.py b/packages/python3-nethsec/tests/test_objects.py new file mode 100644 index 000000000..4ef6a4a79 --- /dev/null +++ b/packages/python3-nethsec/tests/test_objects.py @@ -0,0 +1,380 @@ +import pathlib +from nethsec.utils import ValidationError +import pytest +from euci import EUci, UciExceptionNotFound +from nethsec import firewall, objects + +objects_db = """ +config domain 'myset' + option name 'MySet' + option description 'Mydomain set' + option family 'ipv4' + option timeout '600' + list domain 'www.nethsecurity.org' + list domain 'www.nethserver.org' + +config host 'ns_04fadb5c' + option name 'h1' + option family 'ipv4' + list ipaddr '1.2.3.4' + +config host 'ns_12345' + option name 'h1' + option family 'ipv4' + list ipaddr 'dhcp/ns_112233' + +config host 'ns_a7439990' + option name 'h1v6' + option family 'ipv6' + list ipaddr 'fd:618c:d80a:dc82:2380:54c7:7a17:1013' +""" + +firewall_db = """ +config rule 'r5' + option name 'r5' + option ns_dst 'dhcp/ns_8dcab636' + +config rule 'r6' + option name 'r6' + +config rule 'r7' + option name 'r7' + option ns_src 'objects/ns_04fadb5c' + +config rule 'r8' + option name 'r8' + +config redirect 'redirect1' + option ns_src '' + option ipset 'redirect1_ipset' + +config ipset 'redirect1_ipset' + option name 'redirect1_ipset' + option match 'src_net' + option enabled '1' + list entry '6.7.8.9' + +config redirect 'redirect2' + option ns_src '' +""" + +dhcp_db = """ +config domain 'ns_8bec5896' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' + +config domain 'ns_112233' + option ip '7.8.9.1' + option name 'used_host' + option ns_description 'Used Host' + +config host 'ns_8dcab636' + option ip '192.168.100.5' + option mac 'fe:54:00:6a:50:bf' + option dns '1' + option name 'host2' + option ns_description 'host2' + +config domain 'ns_9e7f705e' + option ip '1.2.3.4' + option name 'test1.domain' + +config host 'ns_271ca281' + option ip '192.168.100.22' + option mac 'fe:54:00:6a:4a:a1' + option dns '1' + option name 'reserve1' + option ns_description 'reserve1' +""" + +user_db = """ +config user 'ns_user1' + option name "john" + option database "main" + option label "John Doe" + option openvpn_ipaddr "10.10.10.22" + +config user 'ns_user2' + option name "user2" +""" + +mwan3_db = """ +config rule 'ns_r1' + option label 'r1' + option use_policy 'ns_default' + option sticky '0' + option proto 'tcp' + option src_ip '1.2.3.4' + option dest_ip '1.1.1.1' + option ns_src 'users/ns_user2' +""" + +dpi_db = """ +config rule + option action 'block' + list application 'netify.twitter' + list source 'dhcp/ns_9e7f705e' + option description 'Block Twitter for user Goofy' + option enabled '1' + +config exemption + option criteria 'dhcp/ns_271ca281' + option description 'Important host' + option enabled '1' + +config rule 'ns_e775b8a7' + option enabled '1' + option device 'br-lan' + option action 'block' + list application 'netify.amazon-prime' + +""" + +@pytest.fixture +def u(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + with tmp_path.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('users').open('w') as fp: + fp.write(user_db) + with tmp_path.joinpath('mwan3').open('w') as fp: + fp.write(mwan3_db) + with tmp_path.joinpath('dpi').open('w') as fp: + fp.write(dpi_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_add_doman_set(u): + id1 = objects.add_domain_set(u, "mydomainset", "ipv4", ["test1.com", "test2.com"]) + assert u.get("objects", id1, "name") == "mydomainset" + assert u.get("objects", id1, "family") == "ipv4" + assert u.get_all("objects", id1, "domain") == ("test1.com", "test2.com") + assert u.get("objects", id1, "timeout") == "660" + + linked = firewall.get_all_linked(u, f"objects/{id1}") + assert linked['firewall'] is not None + assert u.get('firewall', linked['firewall'][0], 'ns_link') == f"objects/{id1}" + assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset" + assert u.get('firewall', linked['firewall'][0], 'family') == 'ipv4' + assert u.get('firewall', linked['firewall'][0], 'timeout') == '660' + + assert linked['dhcp'] is not None + assert u.get('dhcp', linked['dhcp'][0], 'ns_link') == f"objects/{id1}" + assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset",) + assert u.get_all('dhcp', linked['dhcp'][0], 'domain') == ("test1.com", "test2.com") + + id2 = objects.add_domain_set(u, "mydomainset2", "ipv6", ["test3.com", "test4.com"], 300) + assert u.get("objects", id2, "name") == "mydomainset2" + assert u.get("objects", id2, "family") == "ipv6" + assert u.get_all("objects", id2, "domain") == ("test3.com", "test4.com") + assert u.get("objects", id2, "timeout") == "300" + linked = firewall.get_all_linked(u, f"objects/{id2}") + assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset2" + assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset2",) + +def test_edit_domain_set(u): + id = objects.add_domain_set(u, "mydomainset3", "ipv4", ["test1.com", "test2.com"]) + objects.edit_domain_set(u, id, "mydomainset3b", "ipv6", ["test3.com", "test4.com"], 600) + assert u.get("objects", id, "name") == "mydomainset3b" + assert u.get("objects", id, "family") == "ipv6" + assert u.get_all("objects", id, "domain") == ("test3.com", "test4.com") + assert u.get("objects", id, "timeout") == "600" + +def test_delete_domain_set(u): + with pytest.raises(ValidationError): + objects.delete_domain_set(u, "notpresent") + id = objects.add_domain_set(u, "mydomainset4", "ipv4", ["test1.com", "test2.com"]) + assert objects.delete_domain_set(u, id) == id + linked = firewall.get_all_linked(u, f"objects/{id}") + assert linked['firewall'] == [] + assert linked['dhcp'] == [] + id = objects.add_domain_set(u, "mydomainsettd", "ipv4", ["test1.com", "test2.com"]) + u.set('firewall', 'r8', 'ns_dst', f"objects/{id}") + with pytest.raises(ValidationError): + objects.delete_domain_set(u, id) + +def test_is_used_domain_set(u): + id = objects.add_domain_set(u, "used1", "ipv4", ["test1.com", "test2.com"]) + u.set('firewall', 'r5', 'ns_dst', f"objects/{id}") + used, matches = objects.is_used_domain_set(u, id) + assert used + assert matches == ["firewall/r5"] + +def test_list_domain_sets(u): + sets = objects.list_domain_sets(u) + assert len(sets) == 8 + +def test_add_host_set(u): + with pytest.raises(ValidationError): + objects.add_host_set(u, "myhostset", "ipv4", ["a.b.c.d", "e.f.g.h"]) + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "4.5.6.0/24", "192.168.1.3-192.168.1.10"]) + assert u.get("objects", id1, "name") == "myhostset" + assert u.get_all("objects", id1, "ipaddr") == ("1.2.3.4", "4.5.6.0/24", "192.168.1.3-192.168.1.10") + assert u.get("objects", id1, "family") == "ipv4" + id2 = objects.add_host_set(u, "myhostset2", "ipv6", ["2001:db8:3333:4444:5555:6666:7777:8888", "2001:db8::/95", "2001:db8:3333:4444:5555:6666:7777:8888-2001:db8:3333:4444:5555:6666:7777:8890"]) + assert u.get("objects", id2, "name") == "myhostset2" + assert u.get("objects", id2, "family") == "ipv6" + assert u.get_all("objects", id2, "ipaddr") == ("2001:db8:3333:4444:5555:6666:7777:8888", "2001:db8::/95", "2001:db8:3333:4444:5555:6666:7777:8888-2001:db8:3333:4444:5555:6666:7777:8890") + +def test_edit_host_set(u): + id = objects.add_host_set(u, "myhostset3", "ipv4", ["6.7.8.9"]) + objects.edit_host_set(u, id, "myhostset3b", "ipv4", ["1.1.1.1", "2.2.2.2"]) + assert u.get("objects", id, "name") == "myhostset3b" + assert u.get_all("objects", id, "ipaddr") == ("1.1.1.1", "2.2.2.2") + +def test_delete_host_set(u): + with pytest.raises(ValidationError): + objects.delete_host_set(u, "notpresent") + id = objects.add_host_set(u, "myhostset4", "ipv4", ["6.7.8.9"]) + assert objects.delete_host_set(u, id) == id + id = objects.add_host_set(u, "myhostsettd", "ipv4", ["2.2.2.2"]) + u.set('firewall', 'r8', 'ns_dst', f"objects/{id}") + with pytest.raises(ValidationError): + objects.delete_host_set(u, id) + +def test_is_used_host_set(u): + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.1.1.1"]) + u.set('firewall', 'r6', 'ns_src', f"objects/{id}") + used, matches = objects.is_used_host_set(u, id) + assert used + assert matches == ["firewall/r6"] + +def test_list_host_sets(u): + sets = objects.list_host_sets(u) + assert len(sets) == 10 + +def test_is_singleton_host_set(u): + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "5.6.7.8"]) + assert objects.is_singleton_host_set(u, f"objects/{id1}") == False + id2 = objects.add_host_set(u, "myhostset2", "ipv4", ["8.8.8.8"]) + assert objects.is_singleton_host_set(u, f"objects/{id2}") == True + id3 = objects.add_host_set(u, "myhostset3", "ipv4", ["192.168.7.2-192.168.7.3"]) + assert objects.is_singleton_host_set(u, f"objects/{id3}") == False + id4 = objects.add_host_set(u, "myhostset4", "ipv4", ["192.168.0.0/24"]) + assert objects.is_singleton_host_set(u, f"objects/{id4}") == False + assert objects.is_singleton_host_set(u, f"objects/{id4}", allow_cidr=True) + +def test_add_host_set_loopback(u): + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + id2 = objects.add_host_set(u, "myhostset2", "ipv4", [f"objects/{id1}"]) + with pytest.raises(ValidationError): + objects.edit_host_set(u, id1, "myhostset", "ipv4", [f"objects/{id2}"]) + +def test_is_used_object(u): + used, matches = objects.is_used_object(u, "dhcp/ns_8dcab636") + assert used + assert matches == ["firewall/r5"] + assert objects.is_used_object(u, "dhcp/ns_8bec5896")[0] == False + assert objects.is_used_object(u, "users/ns_user2")[0] == True + used, matches = objects.is_used_object(u, "objects/ns_04fadb5c") + assert used + assert matches == ["firewall/r7"] + used, matches = objects.is_used_object(u, "dhcp/ns_112233") + assert used + assert matches == ["objects/ns_12345"] + +def test_object_exists(u): + assert objects.object_exists(u, "users/ns_user1") + assert objects.object_exists(u, "uknown") == False + +def test_get_object(u): + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + obj = objects.get_object(u, id) + +def test_get_object_ips(u): + id0 = objects.add_host_set(u, "myhostset0", "ipv4", ["4.5.6.7"]) + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "dhcp/ns_8bec5896", "users/ns_user1", f"objects/{id0}"]) + ips = objects.get_object_ips(u, f"objects/{id}") + assert set(ips) == set(["1.2.3.4", "7.8.9.1", "10.10.10.22", "4.5.6.7"]) # check with set to ignore order + +def test_is_domain_set(u): + id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + assert objects.is_domain_set(u, f"objects/{id}") == True + assert objects.is_domain_set(u, "dhcp/ns_8dcab636") == False + assert objects.is_domain_set(u, "users/ns_user1") == False + +def test_is_domain(u): + id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + assert objects.is_domain(u, f"objects/{id}") == False + assert objects.is_domain(u, "dhcp/ns_8bec5896") + +def test_is_host(u): + assert objects.is_host(u, "dhcp/ns_8bec5896") == False + assert objects.is_host(u, "dhcp/ns_8dcab636") + +def test_is_vpn_user(u): + assert objects.is_vpn_user(u, "users/ns_user1") + assert objects.is_vpn_user(u, "users/ns_user2") == False + +def test_is_host_set(u): + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + assert objects.is_host_set(u, f"objects/{id}") + assert objects.is_host_set(u, "dhcp/ns_8dcab636") == False + +def test_list_vpn_users(u): + users = objects.list_vpn_users(u) + assert len(users) == 1 + users = objects.list_vpn_users(u, True) + assert 'ipaddr' in users[0] + +def test_list_dns_records(u): + records = objects.list_dns_records(u) + assert len(records) == 3 + records = objects.list_dns_records(u, True) + for r in records: + assert 'ipaddr' in r + +def test_list_dhcp_static_leases(u): + leases = objects.list_dhcp_static_leases(u) + assert len(leases) == 2 + leases = objects.list_dhcp_static_leases(u, True) + for l in leases: + assert 'ipaddr' in l + +def test_list_objects(u): + objs = objects.list_objects(u) + assert len(objs) == len(objects.list_domain_sets(u)) + len(objects.list_host_sets(u)) + len(objects.list_vpn_users(u)) + len(objects.list_dhcp_static_leases(u)) + len(objects.list_dns_records(u)) + objs = objects.list_objects(u, expand=True) + for o in objs: + if o['type'] == 'host_set': + assert 'ipaddr' in o + elif o['type'] == 'domain_set': + assert 'domain' in o + elif o['type'] == 'vpn_user': + assert 'ipaddr' in o + elif o['type'] == 'dhcp_static_lease': + assert 'ipaddr' in o + elif o['type'] == 'dns_record': + assert 'ipaddr' in o + objs = objects.list_objects(u, include_domain_sets=False) + for o in objs: + assert o['type'] != 'domain_set' + objs = objects.list_objects(u, singleton_only=True) + for o in objs: + if o['type'] == 'host_set': + assert o['singleton'] + assert o['subtype'] + +def test_get_reference_info(u): + ref = objects.get_info(u, "dhcp/ns_8dcab636") + assert ref['type'] == 'host' + assert ref['name'] == 'host2' + assert ref['id'] == 'ns_8dcab636' + assert ref['family'] == 'ipv4' + assert objects.get_info(u, "unknown") == None + ref2 = objects.get_info(u, "objects/ns_a7439990") + assert ref2['type'] == 'host' + assert ref2['name'] == 'h1v6' + assert ref2['id'] == 'ns_a7439990' + assert ref2['family'] == 'ipv6' + +def test_is_object_with_invalid_is(u): + assert objects.is_object_id(None) == False + assert objects.is_object_id(1) == False \ No newline at end of file diff --git a/packages/python3-nethsec/tests/test_ovpn.py b/packages/python3-nethsec/tests/test_ovpn.py new file mode 100644 index 000000000..a5424d4c2 --- /dev/null +++ b/packages/python3-nethsec/tests/test_ovpn.py @@ -0,0 +1,63 @@ +import pytest +import ipaddress +from nethsec import ovpn +from euci import EUci + +openvpn_db = """ +config openvpn 'custom_config' + option port '1194' + +config openvpn 'custom_config2' + option port '1300' + option server '10.10.10.0 255.255.255.0' + +config openvpn 'custom_config3' + option port '1301' + option server_bridge '10.8.0.4 255.255.255.0 10.8.0.128 10.8.0.254' + +config openvpn 'custom_config4' + option lport '1302' + +config openvpn 'custom_config5' + option lport '1303' + option ifconfig '10.7.0.1 10.7.0.2' +""" + +def _setup_db(tmp_path): + # setup fake db + with tmp_path.joinpath('openvpn').open('w') as fp: + fp.write(openvpn_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_to_cidr(): + assert ovpn.to_cidr("255.255.255.0") == 24 + +def test_to_netmask(): + assert ovpn.to_netmask(24) == "255.255.255.0" + +def test_is_used_network(tmp_path): + u = _setup_db(tmp_path) + assert ovpn.is_used_network(u, "10.10.10.0/24") == True + assert ovpn.is_used_network(u, "10.8.0.0/24") == True + assert ovpn.is_used_network(u, "10.7.0.0/24") == True + assert ovpn.is_used_network(u, "192.168.1.0/24") == False + +def test_generate_random_network(tmp_path): + u = _setup_db(tmp_path) + assert ipaddress.ip_network(ovpn.generate_random_network(u)).is_private + +def test_opt2cidr(): + assert ovpn.opt2cidr("192.168.1.0 255.255.255.0") == "192.168.1.0/24" + +def test_is_used_port(tmp_path): + u = _setup_db(tmp_path) + assert ovpn.is_used_port(u, 1194) == True + assert ovpn.is_used_port(u, 1300) == True + assert ovpn.is_used_port(u, 1301) == True + assert ovpn.is_used_port(u, 1302) == True + assert ovpn.is_used_port(u, 1303) == True + assert ovpn.is_used_port(u, 1200) == False + +def test_generate_random_port(tmp_path): + u = _setup_db(tmp_path) + assert type(ovpn.generate_random_port(u, 1200, 1210)) == type(1) \ No newline at end of file diff --git a/packages/python3-nethsec/tests/test_users.py b/packages/python3-nethsec/tests/test_users.py new file mode 100644 index 000000000..ab5b087e1 --- /dev/null +++ b/packages/python3-nethsec/tests/test_users.py @@ -0,0 +1,521 @@ +import pytest +from nethsec import utils, users +from euci import EUci, UciExceptionNotFound +from unittest.mock import MagicMock, patch + + +users_db = """ +config local 'main' + option description 'Main local database' + +config local 'second' + option description 'Secondary local database' + +config local 'third' + option description 'Third local database' + +config ldap 'ldap1' + option description 'Remote OpenLDAP server' + option uri 'ldaps://192.168.100.234' + option tls_reqcert 'never' + option base_dn 'dc=directory,dc=nh' + option user_dn 'ou=People,dc=directory,dc=nh' + option user_attr 'uid' + option user_display_attr 'cn' + option starttls '0' + option schema 'rfc2307' + +config ldap 'ldap2' + option description 'Remote OpenLDAP server' + option uri 'ldap://192.168.1.1' + option tls_reqcert 'always' + option base_dn 'dc=directory,dc=nh' + option user_dn 'ou=People,dc=directory,dc=nh' + option user_attr 'uid' + option user_display_attr 'cn' + option starttls '1' + option schema 'rfc2307' + +config ldap 'ad1' + option description 'Remote AD server' + option uri 'ldaps://ad.nethserver.org' + option tls_reqcert 'always' + option base_dn 'dc=ad,dc=nethserver,dc=org' + option user_dn 'cn=users,dc=ad,dc=nethserver,dc=org' + option user_display_attr 'cn' + option user_attr 'uid' + option starttls '0' + option schema 'ad' + +config user 'u1' + option name "goofy" + option database "main" + option description 'Goofy Doe' + list macaddr '52:54:00:9d:3d:e5' + list ipaddr '192.168.100.23' + list domain 'ns_goofy_name' + list host 'ns_goofy_pc' + option openvpn_ipaddr '10.9.9.38' + +config user 'u2' + option name "daisy" + option database "main" + option description "Daisy" + list ipaddr '192.168.100.22' + list ipaddr '2001:db8:3333:4444:5555:6666:7777:8888' + +config user 'u3' + option name "goofy" + option database "second" + option description 'Another Goofy' + +config user 'u4' + option name "goofy" + option database "ldap1" + +config user 'u5' + option name "john" + option database "ad1" + +config group 'g1' + option name "vip" + option database "main" + option description 'Very Important People' + list user 'goofy' + list user 'daisy' + +config group 'g2' + option name "vip" + option database "ldap1" +""" + +dhcp_db = """ +config host 'ns_goofy_pc' + option name 'goofypc' + option mac '00:00:8c:16:b3:bd' + option ip '192.168.100.36' + option dns '1' + +config domain 'ns_goofy_pc_nethserver_org' + option ip '192.168.100.30' + option name 'goofy.nethserver.org' + option description 'Goofy Workstation' +""" + +rpcd_db = "" + +def _setup_db(tmp_path): + # setup fake db + with tmp_path.joinpath('users').open('w') as fp: + fp.write(users_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('rpcd').open('w') as fp: + fp.write(rpcd_db) + + return EUci(confdir=tmp_path.as_posix()) + +def test_get_user_addresses(tmp_path): + u = _setup_db(tmp_path) + (ipv4, ipv6) = users.get_user_addresses(u, 'goofy') + for ip in ipv4: + assert(ip in ["192.168.100.36", "10.9.9.38", "192.168.100.30", "192.168.100.23"]) + (ipv4, ipv6) = users.get_user_addresses(u, 'daisy') + assert(ipv6 == ["2001:db8:3333:4444:5555:6666:7777:8888"]) + +def test_get_user_macs(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_user_macs(u, 'goofy') == ["52:54:00:9d:3d:e5"]) + assert(users.get_user_macs(u, 'daisy') == []) + +def test_get_group_addresses(tmp_path): + u = _setup_db(tmp_path) + (ipv4, ipv6) = users.get_group_addresses(u, 'vip') + for ip in ipv4: + assert(ip in ["192.168.100.36", "10.9.9.38", "192.168.100.30", "192.168.100.23", "192.168.100.22"]) + assert(ipv6 == ["2001:db8:3333:4444:5555:6666:7777:8888"]) + +def test_get_group_macs(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_group_macs(u, 'vip') == ["52:54:00:9d:3d:e5"]) + +def test_get_user_addresses(tmp_path): + u = _setup_db(tmp_path) + (ipv4, ipv6) = users.get_user_addresses(u, 'goofy') + for ip in ipv4: + assert(ip in ["192.168.100.36", "10.9.9.38", "192.168.100.30", "192.168.100.23"]) + (ipv4, ipv6) = users.get_user_addresses(u, 'daisy') + assert(ipv6 == ["2001:db8:3333:4444:5555:6666:7777:8888"]) + +def test_get_user_macs(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_user_macs(u, 'goofy') == ["52:54:00:9d:3d:e5"]) + assert(users.get_user_macs(u, 'daisy') == []) + +def test_get_group_addresses(tmp_path): + u = _setup_db(tmp_path) + (ipv4, ipv6) = users.get_group_addresses(u, 'vip') + for ip in ipv4: + assert(ip in ["192.168.100.36", "10.9.9.38", "192.168.100.30", "192.168.100.23", "192.168.100.22"]) + assert(ipv6 == ["2001:db8:3333:4444:5555:6666:7777:8888"]) + +def test_get_group_macs(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_group_macs(u, 'vip') == ["52:54:00:9d:3d:e5"]) + +def test_get_user_by_name(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_user_by_name(u, 'goofy') == { + "name": "goofy", + "database": "main", + "description": "Goofy Doe", + "macaddr": ["52:54:00:9d:3d:e5"], + "ipaddr": ["192.168.100.23"], + "domain": ["ns_goofy_name"], + "host": ["ns_goofy_pc"], + "openvpn_ipaddr": "10.9.9.38", + "local": True, + "admin": False, + "id": "u1", + }) + + assert(users.get_user_by_name(u, 'goofy', database="second") == { + "name": "goofy", + "description": "Another Goofy", + "database": "second", + "local": True, + "admin": False, + "id": "u3" + }) + + assert(users.get_user_by_name(u, 'daisy', database="second") == None) + +def test_get_group_by_name(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_group_by_name(u, 'vip') == { + "name": "vip", + "database": "main", + "description": "Very Important People", + "user": ["goofy", "daisy"], + "local": True, + "id": "g1" + }) + +def test_list_databases(tmp_path): + db_list = users.list_databases(_setup_db(tmp_path)) + assert {"name": "main", "description": "Main local database", "type": "local"} in db_list + assert {"name": "second", "description": "Secondary local database", "type": "local"} in db_list + assert {"name": "ldap1", "description": "Remote OpenLDAP server", "type": "ldap", "schema": "rfc2307", "uri": "ldaps://192.168.100.234", "used": []} in db_list + assert {"name": "ad1", "description": "Remote AD server", "type": "ldap", "schema": "ad", "uri": "ldaps://ad.nethserver.org", "used": []} in db_list + +def test_add_local_database(tmp_path): + u = _setup_db(tmp_path) + users.add_local_database(u, "test", "Test database") + assert u.get('users', 'test') != None + assert u.get('users', 'test', 'description') == "Test database" + +def test_add_ldap_database(tmp_path): + u = _setup_db(tmp_path) + users.add_ldap_database(u, "testldap", "ldap://1.2.3.4", "ad", "dc=test,dc=org", "cn=users,dc=test,dc=org", "cn", "cn") + assert u.get('users', 'testldap') != None + assert u.get('users', 'testldap', 'uri') == "ldap://1.2.3.4" + assert u.get('users', 'testldap', 'schema') == "ad" + assert u.get('users', 'testldap', 'base_dn') == "dc=test,dc=org" + assert u.get('users', 'testldap', 'user_dn') == "cn=users,dc=test,dc=org" + assert u.get('users', 'testldap', 'user_attr') == "cn" + assert u.get('users', 'testldap', 'user_display_attr') == "cn" + assert u.get('users', 'testldap', 'start_tls') == "0" + assert u.get('users', 'testldap', 'tls_reqcert') == "never" + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap', 'user_bind_dn') + + users.add_ldap_database(u, "testldap2", "ldaps://server.nethserver.org", "rfc2307", "dc=test,dc=org", "ou=People,dc=test,dc=org", "uid", "displayName", start_tls=True, tls_reqcert="always", description="mydesc") + assert u.get('users', 'testldap2') != None + assert u.get('users', 'testldap2', 'description') == "mydesc" + assert u.get('users', 'testldap2', 'uri') == "ldaps://server.nethserver.org" + assert u.get('users', 'testldap2', 'schema') == "rfc2307" + assert u.get('users', 'testldap2', 'base_dn') == "dc=test,dc=org" + assert u.get('users', 'testldap2', 'user_dn') == "ou=People,dc=test,dc=org" + assert u.get('users', 'testldap2', 'user_attr') == "uid" + assert u.get('users', 'testldap2', 'user_display_attr') == "displayName" + assert u.get('users', 'testldap2', 'start_tls') == "1" + assert u.get('users', 'testldap2', 'tls_reqcert') == "always" + + users.add_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test,dc=org", "ou=People,dc=test,dc=org", "uid", "displayName", start_tls=True, tls_reqcert="always", description="mydesc", bind_dn="cn=admin,dc=test,dc=org", bind_password="12=34", user_bind_dn="%u@test.org") + assert u.get('users', 'testldap3') != None + assert u.get('users', 'testldap3', 'bind_dn') == "cn=admin,dc=test,dc=org" + assert u.get('users', 'testldap3', 'bind_password') == "12=34" + assert u.get('users', 'testldap3', 'user_bind_dn') == "%u@test.org" + + +def test_list_users(tmp_path): + u = _setup_db(tmp_path) + user_list = users.list_users(_setup_db(tmp_path), "second") + assert(users.get_user_by_name(u, 'goofy', "second") in user_list) + +def test_edit_ldap_database(tmp_path): + u = _setup_db(tmp_path) + users.edit_ldap_database(u, "testldap2", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2") + assert u.get_all('users', 'testldap2') == { + "description": "mydesc2", + "uri": "ldaps://server.nethserver.org", + "schema": "rfc2307", + "base_dn": "dc=test2,dc=org2", + "user_dn": "dc=test2,dc=org2", + "user_attr": "uid", + "user_display_attr": "cn", + "start_tls": "0", + "tls_reqcert": "never" + } + + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap3', 'bind_dn') + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap3', 'bind_password') + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2", bind_dn="cn=admin2,dc=test,dc=org", bind_password="4567", user_bind_dn="%u@test2.org2") + assert u.get('users', 'testldap3', 'bind_dn') == "cn=admin2,dc=test,dc=org" + assert u.get('users', 'testldap3', 'bind_password') == "4567" + assert u.get('users', 'testldap3', 'user_bind_dn') == "%u@test2.org2" + # check if user_bind_dn is deleted if not passed as parameter + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2", bind_dn="cn=admin2,dc=test,dc=org", bind_password="4567") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap3', 'user_bind_dn') + # add back user_bind_dn, check if it's removed if passed as empty string + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2", bind_dn="cn=admin2,dc=test,dc=org", bind_password="4567", user_bind_dn="%u@test3.org3") + assert u.get('users', 'testldap3', 'user_bind_dn') == "%u@test3.org3" + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2", bind_dn="cn=admin2,dc=test,dc=org", bind_password="4567", user_bind_dn="") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap3', 'user_bind_dn') + + + +def test_delete_ldap_database(tmp_path): + u = _setup_db(tmp_path) + users.delete_ldap_database(u, "ldap1") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'ldap1') + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'u4') + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'g2') + +def test_edit_local_database(tmp_path): + u = _setup_db(tmp_path) + users.edit_local_database(u, "main", "Main local database2") + assert u.get('users', 'main', 'description') == "Main local database2" + +def test_delete_local_database(tmp_path): + u = _setup_db(tmp_path) + users.delete_local_database(u, "main") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'main') == None + with pytest.raises(UciExceptionNotFound) as e: + u.get('users', 'u1') + with pytest.raises(UciExceptionNotFound) as e: + u.get('users', 'u2') + with pytest.raises(UciExceptionNotFound) as e: + u.get('users', 'g1') + +def test_add_local_user(tmp_path): + u = _setup_db(tmp_path) + id = users.add_local_user(u, "t1", password="nethesis", description="mydesc", database="second", extra_fields={"openvpn_ipaddr": "1.2.3.4"}) + user = users.get_user_by_name(u, "t1", "second") + assert user != None + assert users.check_password("nethesis", user.pop("password")) + assert user == { + "name": "t1", + "description": "mydesc", + "database": "second", + "local": True, + "admin": False, + "id": id, + "openvpn_ipaddr": "1.2.3.4" + } + +def test_edit_local_user(tmp_path): + u = _setup_db(tmp_path) + id = users.edit_local_user(u, "t1", password="pass2", description='mydesc2', database="second", extra_fields={"openvpn_ipaddr": "1.2.3.5", "openvpn_enabled": "1"}) + user = users.get_user_by_name(u, "t1", "second") + assert users.check_password("pass2", user.pop("password")) + assert user == { + "name": "t1", + "description": "mydesc2", + "database": "second", + "local": True, + "admin": False, + "id": id, + "openvpn_ipaddr": "1.2.3.5", + "openvpn_enabled": "1" + } + # make sure description is not changed if not passed as parameter + id = users.edit_local_user(u, "t1", password="pass2", database="second", extra_fields={"openvpn_ipaddr": "1.2.3.5", "openvpn_enabled": "1"}) + assert u.get('users', id, 'description') == 'mydesc2' + id = users.edit_local_user(u, "t1", password="pass2", database="second", extra_fields={"openvpn_enabled": "1"}) + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', id, 'openvpn_ipaddr') + +def test_add_local_group(tmp_path): + u = _setup_db(tmp_path) + id = users.add_local_group(u, "group3", ["goofy", "daisy"], "mydesc", database="second") + assert u.get_all('users', id) == { + "name": "group3", + "description": "mydesc", + "database": "second", + "user": ("goofy", "daisy"), + } + +def test_edit_local_group(tmp_path): + u = _setup_db(tmp_path) + id = users.edit_local_group(u, "group3", ["goofy"], "mydesc2", database="second") + assert u.get('users', id, 'description') == 'mydesc2' + assert u.get('users', id, 'user', list=True) == ('goofy',) + +def test_delete_local_user(tmp_path): + u = _setup_db(tmp_path) + assert users.delete_local_user(u, "t1", database="second") + assert users.get_user_by_name(u, "t1", "second") == None + assert users.delete_local_user(u, "goofy", database="second") + group = users.get_group_by_name(u, "group3", "second") + assert group.get('user',[]) == [] + +def test_delete_local_group(tmp_path): + u = _setup_db(tmp_path) + group = users.get_group_by_name(u, "group3", "second") + users.delete_local_group(u, "group3", database="second") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', group['id']) + +def test_add_remote_user(tmp_path): + u = _setup_db(tmp_path) + id = users.add_remote_user(u, "john", "ldap2", extra_fields={"openvpn_ipaddr": "1.2.3.4"}) + u.get('users', id, 'name') == "john" + u.get('users', id, 'database') == "ldap2" + u.get('users', id, 'openvpn_ipaddr') == "1.2.3.4" + +def test_edit_remote_user(tmp_path): + u = _setup_db(tmp_path) + id = users.edit_remote_user(u, "john", "ldap2", extra_fields={"openvpn_enabled": "1"}) + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', id, 'openvpn_ipaddr') + u.get('users', id, 'openvpn_enabled') == "1" + +def test_delete_remote_user(tmp_path): + u = _setup_db(tmp_path) + user = users.get_user_by_name(u, "john", "ldap2") + users.delete_remote_user(u, "john", database="ldap2") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', user['id']) + +def test_shadow_password(): + shadow = users.shadow_password("test") + assert(users.check_password("test", shadow)) + +def test_ldif2users(): + ldif_data = """ +dn: uid=admin,ou=People,dc=directory,dc=nh +uid: admin +cn: admin + +dn: uid=pluto,ou=People,dc=directory,dc=nh +uid: pluto +cn: Pluto Rossi + +# pagedresults: cookie= +""" + assert users.ldif2users(ldif_data) == [{"name": "admin", "description": "admin"},{"name":"pluto", "description": "Pluto Rossi"}] + + ldif_data = """ +# John Doe, Users, domtest1.local +dn: CN=John Doe,CN=Users,DC=domtest1,DC=local +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: user +cn: John Doe +sn: doe +description: myoffice +physicalDeliveryOfficeName: myoffice +givenName: john +distinguishedName: CN=John Doe,CN=Users,DC=domtest1,DC=local +instanceType: 4 +whenCreated: 20240605091444.0Z +whenChanged: 20240702131054.0Z +displayName: John Doe +uSNCreated: 45212 +uSNChanged: 49102 +name: John Doe +objectGUID:: 0JQMYe1R+0uziXQFbmQoJg== +userAccountControl: 512 +badPwdCount: 0 +codePage: 0 +countryCode: 0 +badPasswordTime: 133621586649855275 +lastLogoff: 0 +lastLogon: 133621586960949063 +pwdLastSet: 133632746540757598 +primaryGroupID: 513 +objectSid:: AQUAAAAAAAUVAAAAvh3X9UPJVVW6ljm9VwQAAA== +accountExpires: 9223372036854775807 +logonCount: 0 +sAMAccountName: john.doe +sAMAccountType: 805306368 +userPrincipalName: john.doe@domtest1.local +objectCategory: CN=Person,CN=Schema,CN=Configuration,DC=domtest1,DC=local +dSCorePropagationData: 16010101000000.0Z +""" + assert users.ldif2users(ldif_data, 'sAMAccountName', 'displayName') == [{"name": "john.doe", "description": "John Doe"}] + +def test_set_admin_user(tmp_path): + u = _setup_db(tmp_path) + users.add_local_user(u, "admin2", password="nethesis", description="mydesc", database="third") + admin_id_rpcd = users.set_admin(u, "admin2", "third") + for user in utils.get_all_by_type(u, "rpcd", "login"): + if user == admin_id_rpcd: + assert u.get('rpcd', admin_id_rpcd, "username") == "admin2" + assert users.check_password("nethesis", u.get('rpcd', admin_id_rpcd, "password")) + +def test_is_admin(tmp_path): + u = _setup_db(tmp_path) + assert users.is_admin(u, "admin2") + user = users.get_user_by_name(u, "admin2", 'third') + assert user.get("admin") + +def test_change_admin_password(tmp_path): + u = _setup_db(tmp_path) + local_id = users.edit_local_user(u, "admin2", password="nethesis", description="mydesc", database="third") + logins = utils.get_all_by_type(u, "rpcd", "login") + for l in logins: + if logins[l].get("username") == "admin2": + assert logins[l].get("password") == u.get("users", local_id, "password") + +def test_remove_admin_user(tmp_path): + u = _setup_db(tmp_path) + users.remove_admin(u, "admin2") + found = False + for user in utils.get_all_by_type(u, "rpcd", "login"): + if user.get("username") == "admin2": + found = True + assert not found + +def test_get_database(tmp_path): + u = _setup_db(tmp_path) + assert users.get_database(u, "third") == { + "name": "third", + "description": "Third local database", + "type": "local" + } + + assert users.get_database(u, "ad1") == { + "name": "ad1", + "description": "Remote AD server", + "type": "ldap", + "schema": "ad", + "uri": "ldaps://ad.nethserver.org", + "tls_reqcert": "always", + "base_dn": "dc=ad,dc=nethserver,dc=org", + "user_dn": "cn=users,dc=ad,dc=nethserver,dc=org", + "user_attr": "uid", + "user_display_attr": "cn", + "starttls": "0" + } diff --git a/packages/python3-nethsec/tests/test_utils.py b/packages/python3-nethsec/tests/test_utils.py new file mode 100644 index 000000000..0c9d4abc5 --- /dev/null +++ b/packages/python3-nethsec/tests/test_utils.py @@ -0,0 +1,229 @@ +import pytest +from nethsec import utils +from euci import EUci +from unittest.mock import MagicMock, patch + +# Setup fake ip command output +ip_json='[{"ifindex":9,"ifname":"vnet3","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"virbr2","operstate":"UNKNOWN","group":"default","txqlen":1000,"link_type":"ether","address":"fe:62:31:19:0b:29","broadcast":"ff:ff:ff:ff:ff:ff","addr_info":[{"family":"inet6","local":"fe80::fc62:31ff:fe19:b29","prefixlen":64,"scope":"link","valid_life_time":4294967295,"preferred_life_time":4294967295}]},{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":3,"ifname":"eth1","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":4,"ifname":"eth2","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:75:1c:c1","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":5,"ifname":"eth3","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":6,"ifname":"ifb-dns","flags":["BROADCAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":32,"link_type":"ether","address":"72:79:65:12:07:07","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":7,"ifname":"br-lan","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":9,"ifname":"bond-bond1","flags":["BROADCAST","MULTICAST","MASTER","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":8,"ifname":"tuntunsubnet","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":500,"link_type":"none"}, {"ifindex":69,"ifname":"pppoe-w1","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1492,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":3,"link_type":"ppp"},{"ifindex":20,"link":"eth1","ifname":"eth1.4","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"}]' +mock_ip_stdout = MagicMock() +mock_ip_stdout.configure_mock(**{"stdout": ip_json}) + +test_db = """ +config mytype section1 + option name 'myname1' + option opt2 'value2' + +config mytype section2 + option name 'myname2' + list opt1 'val1' + list opt1 'val2' + option opt2 'value2' + +config mytype2 section3 + option name 'myname3' +""" + +firewall_db = """ +config zone wan1 + option name 'wan' + list network 'wan' + list network 'wan6' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + list device 'eth2.1' + +config zone + option name 'lan' + list network 'lan' + list network 'br-lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list device 'tunrw1' + list device 'ipsec2' +""" + +network_db = """ +config interface lan + option device 'vnet3' + option proto 'static' + +config device + option name 'br-lan' + option type 'bridge' + list ports 'eth0' + +config device + option type '8021q' + option ifname 'eth2' + option vid '1' + option name 'eth2.1' + +config interface 'wan' + option device 'eth1' + option proto 'dhcp' + +config interface 'wan6' + option device 'eth1' + option proto 'dhcpv6' + +config interface 'bond1' + option proto 'bonding' + option ipaddr '10.0.0.22' + option netmask '255.255.255.0' + list slaves 'eth3' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + option all_slaves_active '0' + option link_monitoring 'off' + +config interface 'bond2' + option proto 'bonding' + option ipaddr '10.11.11.22' + option netmask '255.255.255.0' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + list slaves 'eth3.2' + +config device + option ifname 'eth0' + option name 'eth0.1' + option type '8021q' + option vid '45' + +config device 'ns_29a705ea' + option name 'eth3.2' + option type '8021q' + option ifname 'eth3' + option vid '2' +""" + +dedalo_db = """ +config dedalo 'config' + option disabled '0' + option interface 'eth2' +""" + +def _setup_db(tmp_path): + # setup fake db + with tmp_path.joinpath('test').open('w') as fp: + fp.write(test_db) + with tmp_path.joinpath('network').open('w') as fp: + fp.write(network_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('dedalo').open('w') as fp: + fp.write(dedalo_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_sanitize(): + assert utils.sanitize("good") == "good" + assert utils.sanitize("with-dash") == "with_dash" + assert utils.sanitize('$%_()') == '____' + assert utils.sanitize('UPPER') == 'UPPER' + assert utils.sanitize('numb3r') == 'numb3r' + assert utils.sanitize('newline\n') == 'newline' + assert utils.sanitize('newline\r') == 'newline' + +def test_get_id(): + assert utils.get_id('no-good') == 'ns_no_good' + assert utils.get_id('nospace ') == 'ns_nospace' + assert utils.get_id('t1234') == 'ns_t1234' + # str with 97 chars + long_str = "ihTSEf2Y5rl8TX96pWFFPMty9LFgH3GezhVueGoDB6_aaIFhDSKe1ZR64cV41iSVhfrm5wJCUPFfMGx2fBZyhDIW9cl9SCI43" + assert utils.get_id(long_str) == f'ns_{long_str}' + assert utils.get_id(long_str+"123") == f'ns_{long_str}' + +def test_get_id_length(): + assert utils.get_id("123456789012345", 15) == "ns_123456789012" + +def test_get_all_by_type(tmp_path): + u = _setup_db(tmp_path) + records = utils.get_all_by_type(u, 'test', 'mytype') + assert records != None + assert 'section1' in records.keys() + assert 'section2' in records.keys() + assert 'section3' not in records.keys() + assert records['section1']['name'] == 'myname1' + assert records['section2']['name'] == 'myname2' + assert records['section2']['opt1'] == ('val1', 'val2') + +@patch("nethsec.utils.subprocess.run") +def test_get_device_name_valid(mock_run): + # setup mock + mock_run.return_value = mock_ip_stdout + + assert utils.get_device_name("fe:62:31:19:0b:29") == 'vnet3' + +def test_get_device_name_not_valid(): + assert utils.get_device_name("aa:bb:cc:dd:66:55") == None + +@patch("nethsec.utils.subprocess.run") +def test_get_interface_from_mac(mock_run2, tmp_path): + # setup mock + mock_run2.return_value = mock_ip_stdout + u = _setup_db(tmp_path) + + assert utils.get_interface_from_mac(u, "fe:62:31:19:0b:29") == 'lan' + assert utils.get_interface_from_mac(u, "aa:bb:cc:dd:66:55") == None + +def test_get_interface_from_device(tmp_path): + u = _setup_db(tmp_path) + + assert utils.get_interface_from_device(u, "vnet3") == 'lan' + assert utils.get_interface_from_device(u, "vnet4") == None + +def test_get_all_by_option(tmp_path): + u = _setup_db(tmp_path) + return_map = {"section1": u.get_all("test", "section1"), "section2": u.get_all("test", "section2")} + assert(utils.get_all_by_option(u, 'test', 'opt2', 'value2') == return_map) + assert(utils.get_all_by_option(u, 'test', 'opt2', 'value2', deep = False) == list(return_map.keys())) + +def test_get_all_wan_devices(tmp_path): + u = _setup_db(tmp_path) + assert(set(utils.get_all_wan_devices(u)) == set(['eth1', 'eth2.1'])) + +def test_get_all_lan_devices(tmp_path): + u = _setup_db(tmp_path) + assert(set(utils.get_all_lan_devices(u)) == set(['vnet3'])) + +def test_get_random_id(): + id1 = utils.get_random_id() + id2 = utils.get_random_id() + assert len(id1) == 11 + assert id1[0:3] == "ns_" + assert id1 != id2 + +def test_error(): + obj = utils.generic_error("my Error ") + assert obj == {"error": "my_error"} + +def test_validation_error(): + er1 = utils.validation_error("param1", "my Error ", "val1") + assert er1 == {"validation": {"errors": [{"parameter": "param1", "message": "my_error", "value": "val1"}]}} + er2 = utils.validation_error("param1") + assert er2 == {"validation": {"errors": [{"parameter": "param1", "message": "", "value": ""}]}} + +def test_validation_errors(): + er1 = utils.validation_errors([ + ["param1", " my Error ", "val1"], + ["param2", " invalid_value ", False], + ]) + assert er1 == {"validation": {"errors": [{"parameter": "param1", "message": "my_error", "value": "val1"}, {"parameter": "param2", "message": "invalid_value", "value": False}]}} + +@patch("nethsec.utils.subprocess.run") +def test_get_unassigned_devices(mock_run_ip, tmp_path): + # setup mock + mock_run_ip.return_value = mock_ip_stdout + + u = _setup_db(tmp_path) + assert utils.get_unassigned_devices(u) == ['eth0.1'] + +def test_check_password(): + password = "my_password" + shadow = utils.shadow_password(password) + assert utils.check_password(password, shadow) == True + assert utils.check_password("wrong_password", shadow) == False From 84e258c6eab69e326714c4c1653e00cc1cb606e0 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 24 Jul 2026 11:01:05 +0200 Subject: [PATCH 3/7] fix(python3-nethsec): build the test image locally in CI The tests could not pull the published test image (private and not tagged for the branch), so build it locally when it is missing instead. Assisted-by: Claude Code:claude-opus-4-8[1m] --- .github/workflows/python3-nethsec-tests.yml | 9 ++------- packages/python3-nethsec/README.md | 8 +++++--- packages/python3-nethsec/test.sh | 9 +++++++++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python3-nethsec-tests.yml b/.github/workflows/python3-nethsec-tests.yml index c7be82215..60d41855c 100644 --- a/.github/workflows/python3-nethsec-tests.yml +++ b/.github/workflows/python3-nethsec-tests.yml @@ -6,12 +6,10 @@ on: paths: - ".github/workflows/python3-nethsec-tests.yml" - "packages/python3-nethsec/**" - - "!packages/python3-nethsec/builder/**" pull_request: paths: - ".github/workflows/python3-nethsec-tests.yml" - "packages/python3-nethsec/**" - - "!packages/python3-nethsec/builder/**" jobs: run_tests: @@ -20,12 +18,9 @@ jobs: defaults: run: working-directory: packages/python3-nethsec - env: - # The test image is source-independent (source is bind-mounted at runtime). - # Use the 'main' tag on the main branch, 'latest' everywhere else — raw - # github.ref_name is '/merge' on PRs, which is not a valid image tag. - IMAGETAG: ${{ github.ref_name == 'main' && 'main' || 'latest' }} steps: - uses: actions/checkout@v7 + # test.sh builds the test image locally when it is not already present, + # so the tests do not depend on a published (and possibly private) image. - id: test run: ./test.sh diff --git a/packages/python3-nethsec/README.md b/packages/python3-nethsec/README.md index b79e1893a..a8ea2aee9 100644 --- a/packages/python3-nethsec/README.md +++ b/packages/python3-nethsec/README.md @@ -76,9 +76,11 @@ package directory (`packages/python3-nethsec`) execute: ./test.sh ``` -The test container is built and published by the `python3-nethsec-test-image` -GitHub Actions workflow; `test.sh` pulls `ghcr.io/nethserver/nethsecurity/python3-nethsec-test` -(override the tag with the `IMAGETAG` environment variable, default `latest`). +On the first run `test.sh` builds the test image locally (see `Containerfile`), +so no registry access is needed. Delete the image to force a rebuild after +changing the `Containerfile`. The `python3-nethsec-test-image` GitHub Actions +workflow also builds and publishes the image to +`ghcr.io/nethserver/nethsecurity/python3-nethsec-test` for reuse. ## Packaging in NethSecurity diff --git a/packages/python3-nethsec/test.sh b/packages/python3-nethsec/test.sh index 3f8993ec2..5820daed2 100755 --- a/packages/python3-nethsec/test.sh +++ b/packages/python3-nethsec/test.sh @@ -10,4 +10,13 @@ # IMAGE=ghcr.io/nethserver/nethsecurity/python3-nethsec-test IMAGETAG=${IMAGETAG:-latest} + +# Build the test image locally if it is not already available. This keeps the +# tests self-contained: no need to pull a published image or log into a +# registry, in CI or on a developer machine. Remove the image to force a +# rebuild after changing the Containerfile. +if ! podman image exists "${IMAGE}:${IMAGETAG}"; then + "$(dirname "$0")/builder/build.sh" +fi + podman run --rm --tty --volume .:/app:Z "${IMAGE}:${IMAGETAG}" From aee30e2427d4c9581c2a0f5eb7ff55e6c7c56628 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 24 Jul 2026 11:07:47 +0200 Subject: [PATCH 4/7] docs(python3-nethsec): publish API docs within the main documentation Generate the python3-nethsec API reference inside the existing documentation build and drop the separate workflow. Assisted-by: Claude Code:claude-opus-4-8[1m] --- .github/workflows/jekyll-gh-pages.yml | 23 ++++++++- .github/workflows/python3-nethsec-docs.yml | 56 ---------------------- docs/design/python3-nethsec-apidocs.md | 14 ++++++ 3 files changed, 36 insertions(+), 57 deletions(-) delete mode 100644 .github/workflows/python3-nethsec-docs.yml create mode 100644 docs/design/python3-nethsec-apidocs.md diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml index 217ad8e0b..1d6935436 100644 --- a/.github/workflows/jekyll-gh-pages.yml +++ b/.github/workflows/jekyll-gh-pages.yml @@ -5,7 +5,7 @@ on: # Runs on pushes targeting the default branch push: branches: ["main"] - paths: ["docs/**", "packages/**/README.md"] + paths: ["docs/**", "packages/**/README.md", "packages/python3-nethsec/src/**"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -39,6 +39,27 @@ jobs: with: source: ./docs destination: ./_site + # Generate the python3-nethsec API reference into the built site, so it is + # published as a subpath of the existing documentation (/apidocs/python3-nethsec/). + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Generate python3-nethsec API documentation + working-directory: packages/python3-nethsec + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install docutils pydoctor + pydoctor \ + --project-name=python3-nethsec \ + --project-url=https://github.com/$GITHUB_REPOSITORY \ + --html-viewsource-base=https://github.com/$GITHUB_REPOSITORY/tree/$GITHUB_SHA \ + --make-html \ + --html-output="$GITHUB_WORKSPACE/_site/apidocs/python3-nethsec" \ + --project-base-dir="$(pwd)" \ + --docformat=restructuredtext \ + --intersphinx=https://docs.python.org/3/objects.inv \ + ./src/nethsec - name: Upload artifact uses: actions/upload-pages-artifact@v5 diff --git a/.github/workflows/python3-nethsec-docs.yml b/.github/workflows/python3-nethsec-docs.yml deleted file mode 100644 index 1df0a021d..000000000 --- a/.github/workflows/python3-nethsec-docs.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: "python3-nethsec apidocs" - -on: - workflow_dispatch: - push: - branches: - - main - paths: - - ".github/workflows/python3-nethsec-docs.yml" - - "packages/python3-nethsec/src/**" - pull_request: - paths: - - ".github/workflows/python3-nethsec-docs.yml" - - "packages/python3-nethsec/src/**" - -jobs: - build: - name: "Build API documentation" - runs-on: ubuntu-24.04 - defaults: - run: - working-directory: packages/python3-nethsec - steps: - - uses: actions/checkout@v7 - - name: Set up Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: 3.11 - - - name: Install requirements for documentation generation - run: | - python -m pip install --upgrade pip setuptools wheel - python -m pip install docutils pydoctor - - - name: Generate API documentation with pydoctor - run: | - pydoctor \ - --project-name=python3-nethsec \ - --project-url=https://github.com/$GITHUB_REPOSITORY \ - --html-viewsource-base=https://github.com/$GITHUB_REPOSITORY/tree/$GITHUB_SHA \ - --make-html \ - --html-output=./apidocs \ - --project-base-dir="$(pwd)" \ - --docformat=restructuredtext \ - --intersphinx=https://docs.python.org/3/objects.inv \ - ./src/nethsec - - # NOTE: this repository already publishes its GitHub Pages site from - # jekyll-gh-pages.yml, and a repository can have only one Pages source. - # So the API docs are published as a build artifact instead of a second - # (conflicting) Pages deployment. - - name: Upload API documentation artifact - uses: actions/upload-artifact@v4 - with: - name: python3-nethsec-apidocs - path: packages/python3-nethsec/apidocs diff --git a/docs/design/python3-nethsec-apidocs.md b/docs/design/python3-nethsec-apidocs.md new file mode 100644 index 000000000..8b8641b03 --- /dev/null +++ b/docs/design/python3-nethsec-apidocs.md @@ -0,0 +1,14 @@ +--- +layout: default +title: python3-nethsec API +parent: Design +--- + +# python3-nethsec API reference + +The `nethsec` Python library is packaged in this repository under +`packages/python3-nethsec`. Its API reference is generated automatically from +the source with [pydoctor](https://pydoctor.readthedocs.io) each time the +documentation is built. + +[Open the python3-nethsec API reference]({{ site.baseurl }}/apidocs/python3-nethsec/) From ba620a6ce3b41560bfafc350c185ff2f552f7d13 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 24 Jul 2026 12:17:25 +0200 Subject: [PATCH 5/7] docs: update the docs build guide for the container flow and API docs Building the site now needs a Ruby container (host Ruby 4.x cannot build the pinned gems) and a pydoctor step for the python3-nethsec API docs. Assisted-by: Claude Code:claude-opus-4-8[1m] --- AGENTS.md | 12 +++++------ docs/README.md | 57 ++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 531b1fb3a..a0e4eb0e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,13 +42,11 @@ Output: `bin/` (image + packages) and `build-logs/`. **Build system is pure OpenWrt `make`** — no npm, pip, go build, or other language-specific tooling. All derivation runs inside the container. -**Docs preview** (only when a local docs build is explicitly needed): -```bash -cd docs -bundle config set --local path 'vendor/bundle' -bundle install -./prepare.sh && bundle exec jekyll serve -``` +**Docs preview** (only when a local docs build is explicitly needed): see +[`docs/README.md`](docs/README.md). The `github-pages` gems do not build on +recent host Ruby (4.x), so the site is built in a `ruby:3.3` container and the +`python3-nethsec` API docs are generated on top with pydoctor; the built +`_site` is then served statically. --- diff --git a/docs/README.md b/docs/README.md index 901f67b30..29c9220d9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,24 +1,55 @@ # Build the documentation -The documentation is built using Github pages on each commit to the main branch. +The documentation is built and published to GitHub Pages on each commit to the +`main` branch by the `Build documentation` workflow. + +The site includes the auto-generated `python3-nethsec` API reference, produced +with [pydoctor](https://pydoctor.readthedocs.io) and published under +`/apidocs/python3-nethsec/` (linked from *Design → python3-nethsec API*). ## Build locally -Tested on Fedora 36. +The Jekyll dependencies (`github-pages`) pin old gems that do not build on +recent Ruby (4.x). The reliable way to build locally is inside a container with +an older Ruby, then generate the API docs on top of the built site. -Install the dependciesm make sure to install ruby >= 3.0 -``` -dnf module install ruby -dnf install ruby-devel -``` +```bash +cd docs -Install jekyll and all dependencies: +# 1. Build the Jekyll site in a container (avoids host Ruby version issues) +podman run --rm -v "$PWD/..":/repo:Z -w /repo/docs ruby:3.3 bash -c ' + gem install bundler -v 2.5.20 --no-document && + bundle install && + ./prepare.sh && + bundle exec jekyll build' + +# 2. Generate the python3-nethsec API docs into the built site +pip install docutils pydoctor +pydoctor \ + --project-name=python3-nethsec \ + --make-html \ + --html-output=./_site/apidocs/python3-nethsec \ + --project-base-dir="$PWD/../packages/python3-nethsec" \ + --docformat=restructuredtext \ + --intersphinx=https://docs.python.org/3/objects.inv \ + ../packages/python3-nethsec/src/nethsec + +# 3. Serve the built site +python3 -m http.server 4000 --directory ./_site +# open http://localhost:4000 ``` + +The site is served statically (step 3) rather than with `jekyll serve` so that +a rebuild does not wipe the generated API docs. + +### Native build (only if the host Ruby can build the gems) + +On a host with Ruby 3.x you can build without a container: + +```bash +cd docs bundle config set --local path 'vendor/bundle' bundle install -``` - -Build and serve the site locally: -``` -./prepare.sh && bundle exec jekyll serve +./prepare.sh && bundle exec jekyll build +# then run steps 2 and 3 above ``` From 27bb8a48c000200dbc99e23204fd744304d1786d Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Mon, 3 Aug 2026 12:34:12 +0200 Subject: [PATCH 6/7] fix(python3-nethsec): derive wheel version from Makefile PKG_VERSION setup.py hardcoded its own version, duplicating PKG_VERSION in the Makefile and requiring both to stay in sync. Read it from an env var injected by the OpenWrt build instead. Assisted-by: Claude Code:claude-sonnet-5 --- packages/python3-nethsec/Makefile | 3 +++ packages/python3-nethsec/setup.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/python3-nethsec/Makefile b/packages/python3-nethsec/Makefile index 3181419bb..545def12d 100644 --- a/packages/python3-nethsec/Makefile +++ b/packages/python3-nethsec/Makefile @@ -17,6 +17,9 @@ PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_NAME)-$(PKG_VERSION) PYTHON3_PKG_WHEEL_NAME=nethsec PYTHON3_PKG_WHEEL_VERSION:=$(PKG_VERSION) +# setup.py has no hardcoded version: it reads NETHSEC_PKG_VERSION so PKG_VERSION +# above stays the single source of truth for the wheel version. +PYTHON3_PKG_BUILD_VARS:=NETHSEC_PKG_VERSION=$(PKG_VERSION) include $(INCLUDE_DIR)/package.mk include $(TOPDIR)/feeds/packages/lang/python/python3-package.mk diff --git a/packages/python3-nethsec/setup.py b/packages/python3-nethsec/setup.py index 638c5457f..37bfc8bd0 100644 --- a/packages/python3-nethsec/setup.py +++ b/packages/python3-nethsec/setup.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +import os + from setuptools import setup, find_packages with open("README.md", "r") as fh: @@ -7,7 +9,8 @@ setup( name = 'nethsec', - version = '1.7.1', + # Set by the OpenWrt Makefile (PKG_VERSION); falls back for local/dev builds. + version = os.environ.get('NETHSEC_PKG_VERSION', '0.0.0'), author = 'Giacomo Sanchietti', author_email = 'giacomo.sanchietti@nethesis.it', description = 'Utilities for NethSecurity development', From 1050ed8b3a043d1d5666b1b8fe7ea831a9fb07d5 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Mon, 3 Aug 2026 15:02:38 +0200 Subject: [PATCH 7/7] chore(python3-nethsec): drop test image publishing The test image published to ghcr.io was unused: the tests workflow builds it locally. Remove the publish workflow and builder/build.sh, and rebuild the image from the Containerfile on every test.sh run so a changed Containerfile is never shadowed by a stale local image. Assisted-by: Claude Code:claude-opus-5[1m] --- .../workflows/python3-nethsec-test-image.yml | 42 ------------------- .github/workflows/python3-nethsec-tests.yml | 4 +- packages/python3-nethsec/README.md | 8 ++-- packages/python3-nethsec/builder/build.sh | 24 ----------- packages/python3-nethsec/test.sh | 18 ++++---- 5 files changed, 13 insertions(+), 83 deletions(-) delete mode 100644 .github/workflows/python3-nethsec-test-image.yml delete mode 100755 packages/python3-nethsec/builder/build.sh diff --git a/.github/workflows/python3-nethsec-test-image.yml b/.github/workflows/python3-nethsec-test-image.yml deleted file mode 100644 index 7dce02d77..000000000 --- a/.github/workflows/python3-nethsec-test-image.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: "python3-nethsec test image" - -on: - workflow_dispatch: - push: - paths: - - ".github/workflows/python3-nethsec-test-image.yml" - - "packages/python3-nethsec/builder/**" - - "packages/python3-nethsec/Containerfile" - - "packages/python3-nethsec/requirements.txt" - -permissions: - packages: write - -jobs: - publish_images: - name: "Build and publish" - runs-on: ubuntu-24.04 - defaults: - run: - working-directory: packages/python3-nethsec - env: - IMAGETAG: ${{ github.ref_name }} - steps: - - uses: actions/checkout@v7 - - id: build - run: ./builder/build.sh - - id: publish - run: | - # Publish the branch - trap 'buildah logout ghcr.io' EXIT - buildah login -u ${{ github.actor }} --password-stdin ghcr.io <<<"${{ secrets.GITHUB_TOKEN }}" - images=(${{ steps.build.outputs.images }}) - urls="" - for image in "${images[@]}" ; do - buildah push $image docker://${image}:${IMAGETAG:?} - if [[ "${IMAGETAG}" == "main" || "${IMAGETAG}" == "master" ]]; then - buildah push $image docker://${image}:latest - fi - urls="${image}:${IMAGETAG} "$'\n'"${urls}" - done - echo "::notice title=Image URLs::${urls}" diff --git a/.github/workflows/python3-nethsec-tests.yml b/.github/workflows/python3-nethsec-tests.yml index 60d41855c..f0bd86d55 100644 --- a/.github/workflows/python3-nethsec-tests.yml +++ b/.github/workflows/python3-nethsec-tests.yml @@ -20,7 +20,7 @@ jobs: working-directory: packages/python3-nethsec steps: - uses: actions/checkout@v7 - # test.sh builds the test image locally when it is not already present, - # so the tests do not depend on a published (and possibly private) image. + # test.sh always builds the test image locally from the Containerfile, so + # the tests do not depend on a published (and possibly private) image. - id: test run: ./test.sh diff --git a/packages/python3-nethsec/README.md b/packages/python3-nethsec/README.md index a8ea2aee9..ba5ad6429 100644 --- a/packages/python3-nethsec/README.md +++ b/packages/python3-nethsec/README.md @@ -76,11 +76,9 @@ package directory (`packages/python3-nethsec`) execute: ./test.sh ``` -On the first run `test.sh` builds the test image locally (see `Containerfile`), -so no registry access is needed. Delete the image to force a rebuild after -changing the `Containerfile`. The `python3-nethsec-test-image` GitHub Actions -workflow also builds and publishes the image to -`ghcr.io/nethserver/nethsecurity/python3-nethsec-test` for reuse. +`test.sh` rebuilds the test image from the `Containerfile` on every run, so no +registry access is needed and a `Containerfile` change is always picked up. +Podman's layer cache keeps the rebuild fast when nothing changed. ## Packaging in NethSecurity diff --git a/packages/python3-nethsec/builder/build.sh b/packages/python3-nethsec/builder/build.sh deleted file mode 100755 index d628e99b7..000000000 --- a/packages/python3-nethsec/builder/build.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# -# Create ghcr.io/nethserver/nethsecurity/python3-nethsec-test container -# - -set -e - -podman build --force-rm --layers --jobs 0 --tag ghcr.io/nethserver/nethsecurity/python3-nethsec-test . -images+=("ghcr.io/nethserver/nethsecurity/python3-nethsec-test") - -if [[ -n "${CI}" ]]; then - # Set output value for Github Actions - printf "images=%s\n" "${images[*]}" >> "${GITHUB_OUTPUT}" -else - printf "Publish the images with:\n\n" - for image in "${images[@]}"; do printf " buildah push %s docker://%s:latest\n" "${image}" "${image}" ; done - printf "\n" -fi diff --git a/packages/python3-nethsec/test.sh b/packages/python3-nethsec/test.sh index 5820daed2..b80c7ca6f 100755 --- a/packages/python3-nethsec/test.sh +++ b/packages/python3-nethsec/test.sh @@ -8,15 +8,13 @@ # # The container assumes the source code is mounted inside /app # -IMAGE=ghcr.io/nethserver/nethsecurity/python3-nethsec-test -IMAGETAG=${IMAGETAG:-latest} +IMAGE=python3-nethsec-test -# Build the test image locally if it is not already available. This keeps the -# tests self-contained: no need to pull a published image or log into a -# registry, in CI or on a developer machine. Remove the image to force a -# rebuild after changing the Containerfile. -if ! podman image exists "${IMAGE}:${IMAGETAG}"; then - "$(dirname "$0")/builder/build.sh" -fi +set -e -podman run --rm --tty --volume .:/app:Z "${IMAGE}:${IMAGETAG}" +# Always rebuild from the Containerfile: podman's layer cache makes this cheap +# when nothing changed, and a modified Containerfile can never be shadowed by a +# stale image. +podman build --force-rm --layers --jobs 0 --tag "${IMAGE}" "$(dirname "$0")" + +podman run --rm --tty --volume .:/app:Z "${IMAGE}"