diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml index 217ad8e0b..1d6935436 100644 --- a/.github/workflows/jekyll-gh-pages.yml +++ b/.github/workflows/jekyll-gh-pages.yml @@ -5,7 +5,7 @@ on: # Runs on pushes targeting the default branch push: branches: ["main"] - paths: ["docs/**", "packages/**/README.md"] + paths: ["docs/**", "packages/**/README.md", "packages/python3-nethsec/src/**"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -39,6 +39,27 @@ jobs: with: source: ./docs destination: ./_site + # Generate the python3-nethsec API reference into the built site, so it is + # published as a subpath of the existing documentation (/apidocs/python3-nethsec/). + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Generate python3-nethsec API documentation + working-directory: packages/python3-nethsec + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install docutils pydoctor + pydoctor \ + --project-name=python3-nethsec \ + --project-url=https://github.com/$GITHUB_REPOSITORY \ + --html-viewsource-base=https://github.com/$GITHUB_REPOSITORY/tree/$GITHUB_SHA \ + --make-html \ + --html-output="$GITHUB_WORKSPACE/_site/apidocs/python3-nethsec" \ + --project-base-dir="$(pwd)" \ + --docformat=restructuredtext \ + --intersphinx=https://docs.python.org/3/objects.inv \ + ./src/nethsec - name: Upload artifact uses: actions/upload-pages-artifact@v5 diff --git a/.github/workflows/python3-nethsec-scans.yml b/.github/workflows/python3-nethsec-scans.yml new file mode 100644 index 000000000..edc29f70e --- /dev/null +++ b/.github/workflows/python3-nethsec-scans.yml @@ -0,0 +1,49 @@ +name: "python3-nethsec scans" + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - ".github/workflows/python3-nethsec-scans.yml" + - "packages/python3-nethsec/**" + pull_request: + paths: + - ".github/workflows/python3-nethsec-scans.yml" + - "packages/python3-nethsec/**" + +permissions: + contents: read + security-events: write + +jobs: + sbom: + name: SBOM + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - name: Generate SPDX SBOM + uses: aquasecurity/trivy-action@0.35.0 + with: + scan-type: fs + scan-ref: packages/python3-nethsec + format: spdx-json + output: sbom.spdx.json + - name: Generate Sarif Report + uses: aquasecurity/trivy-action@0.35.0 + with: + scan-type: fs + scan-ref: packages/python3-nethsec + format: sarif + output: trivy.sarif.json + - name: Upload report to GitHub + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: trivy.sarif.json + category: trivy-python3-nethsec + - name: Upload SBOM artifact + uses: actions/upload-artifact@v4 + with: + name: python3-nethsec-sbom + path: sbom.spdx.json diff --git a/.github/workflows/python3-nethsec-tests.yml b/.github/workflows/python3-nethsec-tests.yml new file mode 100644 index 000000000..f0bd86d55 --- /dev/null +++ b/.github/workflows/python3-nethsec-tests.yml @@ -0,0 +1,26 @@ +name: "python3-nethsec tests" + +on: + workflow_dispatch: + push: + paths: + - ".github/workflows/python3-nethsec-tests.yml" + - "packages/python3-nethsec/**" + pull_request: + paths: + - ".github/workflows/python3-nethsec-tests.yml" + - "packages/python3-nethsec/**" + +jobs: + run_tests: + name: "Run tests" + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: packages/python3-nethsec + steps: + - uses: actions/checkout@v7 + # test.sh always builds the test image locally from the Containerfile, so + # the tests do not depend on a published (and possibly private) image. + - id: test + run: ./test.sh diff --git a/AGENTS.md b/AGENTS.md index a441b31be..a0e4eb0e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,13 +42,11 @@ Output: `bin/` (image + packages) and `build-logs/`. **Build system is pure OpenWrt `make`** — no npm, pip, go build, or other language-specific tooling. All derivation runs inside the container. -**Docs preview** (only when a local docs build is explicitly needed): -```bash -cd docs -bundle config set --local path 'vendor/bundle' -bundle install -./prepare.sh && bundle exec jekyll serve -``` +**Docs preview** (only when a local docs build is explicitly needed): see +[`docs/README.md`](docs/README.md). The `github-pages` gems do not build on +recent host Ruby (4.x), so the site is built in a `ruby:3.3` container and the +`python3-nethsec` API docs are generated on top with pydoctor; the built +`_site` is then served statically. --- @@ -199,7 +197,6 @@ Existing Python files (listed in `ruff.toml` under `extend-exclude`) are grandfa | Repo | Purpose | |---|---| -| [NethServer/nethsecurity-api](https://github.com/NethServer/nethsecurity-api) | Go API server (ns-api-server); handles JWT auth, 2FA, proxies to ubus/rpcd. **Vendored in-tree**: source lives at `packages/ns-api-server/files/src/` and is built directly from there (`Build/Prepare` copies it, no upstream fetch) — edit it in this repo, then mirror the change upstream to keep the two in sync | | [NethServer/python3-nethsec](https://github.com/NethServer/python3-nethsec) | Python utility library for APIs; used by all Python API scripts | | [NethServer/nethsecurity-ui](https://github.com/NethServer/nethsecurity-ui) | Vue 3 + Tailwind CSS management UI (standalone + controller modes); built as `ns-ui` package | | [NethServer/nethsecurity-monitoring](https://github.com/NethServer/nethsecurity-monitoring) | Go monitoring tools (ns-flows); reads netifyd flows, exposes REST API on localhost:8080 | diff --git a/docs/README.md b/docs/README.md index 901f67b30..29c9220d9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,24 +1,55 @@ # Build the documentation -The documentation is built using Github pages on each commit to the main branch. +The documentation is built and published to GitHub Pages on each commit to the +`main` branch by the `Build documentation` workflow. + +The site includes the auto-generated `python3-nethsec` API reference, produced +with [pydoctor](https://pydoctor.readthedocs.io) and published under +`/apidocs/python3-nethsec/` (linked from *Design → python3-nethsec API*). ## Build locally -Tested on Fedora 36. +The Jekyll dependencies (`github-pages`) pin old gems that do not build on +recent Ruby (4.x). The reliable way to build locally is inside a container with +an older Ruby, then generate the API docs on top of the built site. -Install the dependciesm make sure to install ruby >= 3.0 -``` -dnf module install ruby -dnf install ruby-devel -``` +```bash +cd docs -Install jekyll and all dependencies: +# 1. Build the Jekyll site in a container (avoids host Ruby version issues) +podman run --rm -v "$PWD/..":/repo:Z -w /repo/docs ruby:3.3 bash -c ' + gem install bundler -v 2.5.20 --no-document && + bundle install && + ./prepare.sh && + bundle exec jekyll build' + +# 2. Generate the python3-nethsec API docs into the built site +pip install docutils pydoctor +pydoctor \ + --project-name=python3-nethsec \ + --make-html \ + --html-output=./_site/apidocs/python3-nethsec \ + --project-base-dir="$PWD/../packages/python3-nethsec" \ + --docformat=restructuredtext \ + --intersphinx=https://docs.python.org/3/objects.inv \ + ../packages/python3-nethsec/src/nethsec + +# 3. Serve the built site +python3 -m http.server 4000 --directory ./_site +# open http://localhost:4000 ``` + +The site is served statically (step 3) rather than with `jekyll serve` so that +a rebuild does not wipe the generated API docs. + +### Native build (only if the host Ruby can build the gems) + +On a host with Ruby 3.x you can build without a container: + +```bash +cd docs bundle config set --local path 'vendor/bundle' bundle install -``` - -Build and serve the site locally: -``` -./prepare.sh && bundle exec jekyll serve +./prepare.sh && bundle exec jekyll build +# then run steps 2 and 3 above ``` diff --git a/docs/design/python3-nethsec-apidocs.md b/docs/design/python3-nethsec-apidocs.md new file mode 100644 index 000000000..8b8641b03 --- /dev/null +++ b/docs/design/python3-nethsec-apidocs.md @@ -0,0 +1,14 @@ +--- +layout: default +title: python3-nethsec API +parent: Design +--- + +# python3-nethsec API reference + +The `nethsec` Python library is packaged in this repository under +`packages/python3-nethsec`. Its API reference is generated automatically from +the source with [pydoctor](https://pydoctor.readthedocs.io) each time the +documentation is built. + +[Open the python3-nethsec API reference]({{ site.baseurl }}/apidocs/python3-nethsec/) diff --git a/packages/python3-nethsec/.gitignore b/packages/python3-nethsec/.gitignore new file mode 100644 index 000000000..59bb9ba78 --- /dev/null +++ b/packages/python3-nethsec/.gitignore @@ -0,0 +1,2 @@ +**/__pycache__/ +*.cpython* diff --git a/packages/python3-nethsec/Containerfile b/packages/python3-nethsec/Containerfile new file mode 100644 index 000000000..ed662b1f3 --- /dev/null +++ b/packages/python3-nethsec/Containerfile @@ -0,0 +1,33 @@ +FROM python:3.13.9 +RUN apt-get update \ + && apt-get install -y \ + cmake \ + liblua5.1-0-dev \ + lua5.1 \ + libjson-c-dev + +RUN mkdir /tmp/requirements \ + && git clone https://github.com/openwrt/libubox.git /tmp/requirements/libubox \ + && cd /tmp/requirements/libubox \ + && git checkout 815633847cd32ffe6da28943cbeb37edc88265c8 \ + && cmake CMakeLists.txt \ + && make install \ + && git clone https://github.com/openwrt/ubus.git /tmp/requirements/ubus \ + && cd /tmp/requirements/ubus \ + && git checkout 3cc98db1a422dcf560f2d6347fd410f17565a89d \ + && cmake CMakeLists.txt \ + && make install \ + && git clone https://github.com/openwrt/uci.git /tmp/requirements/uci \ + && cd /tmp/requirements/uci \ + && git checkout 66127cd76c5d0bd46d5a90302cc6110f53a4e2f8 \ + && cmake CMakeLists.txt \ + && make install \ + && rm -rf /tmp/requirements \ + && echo "/usr/local/lib" >> /etc/ld.so.conf.d/local.conf \ + && ldconfig + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm /tmp/requirements.txt +WORKDIR /app +CMD ["python3", "-m", "pytest"] diff --git a/packages/python3-nethsec/LICENSE b/packages/python3-nethsec/LICENSE new file mode 100644 index 000000000..f288702d2 --- /dev/null +++ b/packages/python3-nethsec/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/packages/python3-nethsec/Makefile b/packages/python3-nethsec/Makefile index 54804f55f..545def12d 100644 --- a/packages/python3-nethsec/Makefile +++ b/packages/python3-nethsec/Makefile @@ -6,22 +6,20 @@ include $(TOPDIR)/rules.mk PKG_NAME:=python3-nethsec -# renovate: datasource=github-tags depName=NethServer/python3-nethsec PKG_VERSION:=1.7.1 PKG_RELEASE:=1 - + PKG_MAINTAINER:=Giacomo Sanchietti PKG_LICENSE:=GPL-3.0-only +PKG_LICENSE_FILES:=LICENSE -PKG_SOURCE_PROTO:=git -PKG_SOURCE_URL:=https://github.com/NethServer/python3-nethsec.git -PKG_SOURCE_VERSION:=$(PKG_VERSION) -PKG_SOURCE_SUBDIR:=python3-nethsec-$(PKG_SOURCE_VERSION) -PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) -PKG_MIRROR_HASH:=skip +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_NAME)-$(PKG_VERSION) PYTHON3_PKG_WHEEL_NAME=nethsec PYTHON3_PKG_WHEEL_VERSION:=$(PKG_VERSION) +# setup.py has no hardcoded version: it reads NETHSEC_PKG_VERSION so PKG_VERSION +# above stays the single source of truth for the wheel version. +PYTHON3_PKG_BUILD_VARS:=NETHSEC_PKG_VERSION=$(PKG_VERSION) include $(INCLUDE_DIR)/package.mk include $(TOPDIR)/feeds/packages/lang/python/python3-package.mk @@ -30,15 +28,26 @@ define Package/python3-nethsec SECTION:=base CATEGORY:=NethSecurity TITLE:=NethSecurity python libraries - URL:=https://github.com/NethServer/python3-nethsec/ + URL:=https://github.com/NethServer/nethsecurity/tree/master/packages/python3-nethsec DEPENDS:=+python3-uci +python3-libpass PKGARCH:=all endef - + define Package/python3-nethsec/description NethSecurity python utilities endef +# Source lives in this repository under ./src, so build the wheel from the +# local tree instead of fetching it. Only the files needed to build the wheel +# are copied; tests/ and the CI helpers are intentionally left out. +define Build/Prepare + mkdir -p $(PKG_BUILD_DIR) + $(CP) ./src $(PKG_BUILD_DIR)/ + $(CP) ./setup.py $(PKG_BUILD_DIR)/ + # setup.py reads README.md for the wheel long_description + $(CP) ./README.md $(PKG_BUILD_DIR)/ +endef + $(eval $(call Py3Package,python3-nethsec)) $(eval $(call BuildPackage,python3-nethsec)) $(eval $(call BuildPackage,python3-nethsec-src)) diff --git a/packages/python3-nethsec/README.md b/packages/python3-nethsec/README.md index 2f88b90ef..ba5ad6429 100644 --- a/packages/python3-nethsec/README.md +++ b/packages/python3-nethsec/README.md @@ -1,5 +1,95 @@ -# ns-python3-nethsec +# python3-nethsec -[python3-nethsec](https://github.com/NethServer/python3-nethsec) utilities libraries. +Python3 library for NethSecurity -See online documentation: [https://nethserver.github.io/python3-nethsec/](https://nethserver.github.io/python3-nethsec/). +Requirements: + +* Python3 +* [pyuci](https://gitlab.nic.cz/turris/pyuci) + +[![Run tests](https://github.com/NethServer/nethsecurity/actions/workflows/python3-nethsec-tests.yml/badge.svg?branch=main)](https://github.com/NethServer/nethsecurity/actions/workflows/python3-nethsec-tests.yml) + +## Usage + +The `nethsec` library is composed by the following sub-packages: + +- utils +- firewall + +Usage example: +```python +from euci import EUci +from nethsec import firewall + +u = EUci() +firewall.add_to_lan(u, 'tunrw') +firewall.add_service(u, 'openvpn_rw', '1194', ['udp', 'tcp']) +firewall.apply(u) +``` + +## Documentation + +Documentation can be generated using [pydoctor](https://pydoctor.readthedocs.io) or [pydoc](https://docs.python.org/3/library/pydoc.html). + +Documentation is automatically generated at each new commit on master branch. +Online doc is hosted on [GitHub pages](https://nethserver.github.io/python3-nethsec). + +Generate doc using pydoctor: +``` +pip install -U pydoctor +pydoctor \ + --project-name=python3-nethsec \ + --project-url=https://github.com/nethserver/python3-nethsec \ + --make-html \ + --html-output=./apidocs \ + --project-base-dir="$(pwd)" \ + --docformat=restructuredtext \ + --intersphinx=https://docs.python.org/3/objects.inv \ + ./src/nethsec +``` + +Generate doc using pydoc: +```bash +cd src +python3 -m pydoc nethsec.firewall +python3 -m pydoc nethsec.utils +``` + +## Build + +Execute: +```bash +python3 -m pip install --upgrade build +python3 -m pip install wheel +python3 -m build +``` + +## Tests + +Tests run inside a podman container (see `Containerfile`) based on Python 3.13, +with the OpenWrt `libubox`/`ubus`/`uci` C libraries and the test dependencies +baked in. + +To start the tests, first make sure to have podman installed. Then, from this +package directory (`packages/python3-nethsec`) execute: +``` +./test.sh +``` + +`test.sh` rebuilds the test image from the `Containerfile` on every run, so no +registry access is needed and a `Containerfile` change is always picked up. +Podman's layer cache keeps the rebuild fast when nothing changed. + +## Packaging in NethSecurity + +This library is built into the NethSecurity image as the `python3-nethsec` +OpenWrt package. The `nethsec` wheel is built from the local `./src` tree — the +source is **not** fetched from an external repository. + +When releasing a new version, bump **both** in lockstep: + +- `PKG_VERSION` (and `PKG_RELEASE`) in `Makefile` +- `version` in `setup.py` + +`PYTHON3_PKG_WHEEL_VERSION` uses `PKG_VERSION` to locate the wheel produced by +`setup.py`, so the two values must match or the install step will fail. diff --git a/packages/python3-nethsec/pytest.ini b/packages/python3-nethsec/pytest.ini new file mode 100644 index 000000000..ad5c7cc88 --- /dev/null +++ b/packages/python3-nethsec/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = . src diff --git a/packages/python3-nethsec/requirements.txt b/packages/python3-nethsec/requirements.txt new file mode 100644 index 000000000..d4d4e9c58 --- /dev/null +++ b/packages/python3-nethsec/requirements.txt @@ -0,0 +1,6 @@ +coverage==7.6.12 +pydoctor==24.11.2 +pytest==8.3.5 +pytest-mock==3.14.0 +pyuci==0.10.3 +libpass==1.9.3 diff --git a/packages/python3-nethsec/setup.py b/packages/python3-nethsec/setup.py new file mode 100644 index 000000000..37bfc8bd0 --- /dev/null +++ b/packages/python3-nethsec/setup.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import os + +from setuptools import setup, find_packages + +with open("README.md", "r") as fh: + long_description = fh.read() + +setup( + name = 'nethsec', + # Set by the OpenWrt Makefile (PKG_VERSION); falls back for local/dev builds. + version = os.environ.get('NETHSEC_PKG_VERSION', '0.0.0'), + author = 'Giacomo Sanchietti', + author_email = 'giacomo.sanchietti@nethesis.it', + description = 'Utilities for NethSecurity development', + long_description = long_description, + long_description_content_type = "text/markdown", + url = "https://github.com/NethServer/python3-nethsec", + license = "GPLv3", + package_dir = {'': 'src'}, + packages = ['nethsec', 'nethsec.utils', 'nethsec.firewall', 'nethsec.mwan', 'nethsec.dpi', 'nethsec.ipsec', 'nethsec.ovpn', 'nethsec.users', 'nethsec.reverse_proxy', 'nethsec.inventory', 'nethsec.conntrack', 'nethsec.ldif', 'nethsec.objects', 'nethsec.snort'], + requires = [ "pyuci" ], + classifiers = [ + "Programming Language :: Python :: 3", + "License v3 (LGPLv3) License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Operating System :: OS Independent", + ], + python_requires = '>=3.13', +) diff --git a/packages/python3-nethsec/src/nethsec/__init__.py b/packages/python3-nethsec/src/nethsec/__init__.py new file mode 100644 index 000000000..c7c337877 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/__init__.py @@ -0,0 +1,3 @@ +''' +NethSecurity python library +''' diff --git a/packages/python3-nethsec/src/nethsec/conntrack/__init__.py b/packages/python3-nethsec/src/nethsec/conntrack/__init__.py new file mode 100644 index 000000000..8712432fb --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/conntrack/__init__.py @@ -0,0 +1,137 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Library for reading and managing network connections. +""" +import subprocess + +from xml.etree import ElementTree +from xml.etree.ElementTree import Element + + +def __parse_connection_info(flow: Element) -> dict: + """ + Parse the connection information from a flow tag. + + Args: + - flow: ElementTree.Element with the flow tag. + + Returns: + dictionary with the connection information. + """ + result = {} + # expand meta tags + for child in flow.findall('meta'): + if child.get('direction') == 'original': + layer3 = child.find('layer3') + result['source'] = layer3.find('src').text + result['destination'] = layer3.find('dst').text + layer4 = child.find('layer4') + result['protocol'] = layer4.get('protoname') + if layer4.find('sport') is not None: + result['source_port'] = layer4.find('sport').text + if layer4.find('dport') is not None: + result['destination_port'] = layer4.find('dport').text + counters = child.find('counters') + if counters is not None: + result['source_stats'] = { + 'packets': int(counters.find('packets').text), + 'bytes': int(counters.find('bytes').text) + } + if child.get('direction') == 'reply': + counters = child.find('counters') + if counters is not None: + result['destination_stats'] = { + 'packets': int(counters.find('packets').text), + 'bytes': int(counters.find('bytes').text) + } + if child.get('direction') == 'independent': + result['id'] = child.find('id').text + if child.find('timeout') is not None: + result['timeout'] = child.find('timeout').text + if child.find('unreplied') is not None: + result['unreplied'] = True + if child.find('state') is not None: + result['state'] = child.find('state').text + result['labels'] = [] + for label in child.findall('labels/label'): + result['labels'].append(label.text) + + return result + + +def list_connections(labels: list = None): + """ + List all network connections. + + Args: + - labels: optional list of label strings to filter by. Only connections that have + ALL the specified labels are returned. If ``None`` or empty, all + connections are returned. + + Returns: + list of connections. + """ + cmd = ["conntrack", "-L", "-o", "labels,xml"] + if labels: + cmd.extend(["-l", ",".join(labels)]) + result = subprocess.run(cmd, capture_output=True, text=True) + root = ElementTree.fromstring(result.stdout) + connections = [] + for flow in root.findall('flow'): + connections.append(__parse_connection_info(flow)) + + return connections + + +def drop_connection(connection_id: str): + """ + Drop a connection by its id. + + Args: + - connection_id: id of the connection to drop. + + Raises: + - ValueError: if the connection with the given id is not found. + - RuntimeError: if the connection could not be dropped. + """ + connections = list(filter(lambda x: x['id'] == connection_id, list_connections())) + if len(connections) <= 0: + raise ValueError(f"Connection with id {connection_id} not found.") + + connection = connections[0] + process_commands = [ + 'conntrack', + '-D', + '-p', + connection['protocol'], + '-s', + connection['source'], + '-d', + connection['destination'] + ] + if connection['protocol'] not in ['icmp', 'gre']: + process_commands.extend(['--sport', connection['source_port'], '--dport', connection['destination_port']]) + + try: + subprocess.run(process_commands, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Error running command: {e}") + + +def drop_all_connections(): + """ + Flush all connections. + + Raises: + - RuntimeError: if command failed to execute. + """ + try: + subprocess.run(['conntrack', '-F'], check=True, capture_output=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Error running command: {e}") diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py new file mode 100644 index 000000000..5de35668e --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -0,0 +1,496 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Library that handles the DPI rules. +""" + +import json +import subprocess +from fnmatch import fnmatch + +import math +from euci import EUci + +from nethsec import utils, firewall +from nethsec.utils import ValidationError + + +def __load_applications() -> dict[int, str]: + """ + Reads the applications from the netify-apps.conf file. + + Returns: + dict of applications, each dict contains the property "id" and "name" + """ + applications = dict[int, str]() + with open('/etc/netifyd/netify-apps.conf', 'r') as file: + for line in file.readlines(): + if line.startswith('app'): + line_split = line.strip().removesuffix('\n').removeprefix('app:').split(":") + applications[int(line_split[0])] = line_split[1] + return applications + + +def __load_application_categories() -> dict[int, dict[str]]: + """ + Reads the application categories from the netify-categories.json file. + + Returns: + dict of application categories, each dict contains the property "id" and "name" + """ + categories = dict[int, dict[str]]() + with open('/etc/netifyd/netify-categories.json', 'r') as file: + categories_file = json.load(file) + + categories_names = dict[int, str]() + if 'application_tag_index' not in categories_file: + for category_name, applications in categories_file['application_index'].items(): + for application in applications: + categories[application] = { + 'name': category_name + } + else: + categories_application_tag_index: dict[str, int] = categories_file['application_tag_index'] + for category_name, category_id in categories_application_tag_index.items(): + categories_names[category_id] = category_name + + categories_application_index: list[int, list[int]] = categories_file['application_index'] + for category_id, applications_id in categories_application_index: + for application_id in applications_id: + categories[application_id] = { + 'name': categories_names[category_id] + } + + return categories + + +def load_protocols() -> dict[int, str]: + """ + Reads the protocols from the netifyd --dump-protos command. + + Returns: + dict of protocols, each dict contains the property "id" and "name" + """ + result = subprocess.run(['netifyd', '--dump-protos'], check=True, capture_output=True) + protocols = dict[int, str]() + for line in result.stdout.decode().splitlines(): + # lines can be empty + if len(line) < 1: + continue + line_split = line.split(":") + protocols[int(line_split[0].strip())] = line_split[1].strip() + + return protocols + + +def __load_protocol_categories() -> dict[int, dict[str]]: + """ + Reads the protocol categories from the netify-categories.json file. + + Returns: + dict of protocol categories, each dict contains the property "id" and "name" + """ + categories = dict[int, dict[str]]() + with open('/etc/netifyd/netify-categories.json', 'r') as file: + categories_file = json.load(file) + + categories_names = dict[int, str]() + + if 'protocol_tag_index' not in categories_file: + for category_name, protocols in categories_file['protocol_index'].items(): + for protocol in protocols: + categories[protocol] = { + 'name': category_name + } + else: + categories_protocol_tag_index: dict[str, int] = categories_file['protocol_tag_index'] + for category_name, category_id in categories_protocol_tag_index.items(): + categories_names[category_id] = category_name + + categories_protocol_index: list[int, list[int]] = categories_file['protocol_index'] + for category_id, protocol_ids in categories_protocol_index: + for protocol_id in protocol_ids: + categories[protocol_id] = { + 'name': categories_names[category_id] + } + + return categories + + +def __load_blocklist() -> list[dict[str]]: + """ + Format the applications and protocols into a list of dicts. + + Returns: + list of dicts, each dict contains the property "id", "name", "type" and "category" + """ + result = list[dict[str]]() + applications = __load_applications() + application_categories = __load_application_categories() + + for application_id, application_name in applications.items(): + result_application = { + 'id': application_id, + 'name': application_name, + 'type': 'application' + } + if application_id in application_categories: + result_application['category'] = application_categories[application_id] + result.append(result_application) + + protocols = load_protocols() + protocol_categories = __load_protocol_categories() + + for protocol_id, protocol_name in protocols.items(): + result_protocol = { + 'id': protocol_id, + 'name': protocol_name, + 'type': 'protocol' + } + if protocol_id in protocol_categories: + result_protocol['category'] = protocol_categories[protocol_id] + result.append(result_protocol) + + return result + + +def list_devices(e_uci: EUci): + """ + List device-interface available for filtering. + + Returns: + list of dicts, each dict contains the property "interface" and "device" + """ + instance_name = list(e_uci.get('netifyd').keys())[0] + devices = e_uci.get('netifyd', instance_name, 'internal_if', default=[], list=True) + for zone in firewall.list_zones(e_uci).values(): + if zone['name'] == 'wan': + continue + network_devices = utils.get_all_devices_by_zone(e_uci, zone['name']) + devices = list(set(list(devices) + network_devices)) + ret = [] + for item in devices: + interface_name = utils.get_interface_from_device(e_uci, item) + ret.append({ + 'interface': interface_name if interface_name is not None else item, + 'device': item + }) + return ret + +def list_applications(search: str = None, limit: int = None, page: int = 1) -> dict: + """ + List applications available for filtering. + + Args: + - search: search string + - limit: limit the number of results + - page: page number + + Returns: + list of dicts, each dict contains the property "code" and "name" + """ + result = __load_blocklist() + + if search is not None: + # lower string so we can do a case-insensitive search + search = search.lower() + # I'm aware it's far from a readable code, but list comprehension is the fastest way to filter. + result = [item for item in result if + item.get('name', '').lower().find(search) != -1 or + item.get('category', {}).get('name', '').lower().find(search) != -1] + + total = len(result) + + if limit is not None: + result = result[limit * (page - 1):limit * page] + last_page = math.ceil(total / limit) + else: + last_page = 1 + + return { + 'data': result, + 'meta': { + 'last_page': last_page, + 'total': total, + } + } + + +def list_popular(e_uci: EUci, limit: int = None, page: int = 1) -> dict: + """ + List popular applications available for filtering. + + Args: + - limit: limit the number of results + - page: page number + + Returns: + list of dicts, each dict contains the property "id", "name", "type" and "category" + """ + popular_filters = e_uci.get('dpi', 'config', 'popular_filters', default=[], list=True) + block_list = {block['name']: block for block in __load_blocklist()} + result = [] + + for popular_filter in popular_filters: + if popular_filter in block_list.keys(): + result.append(block_list[popular_filter] | {'missing': False}) + else: + result.append({ + 'name': popular_filter, + 'missing': True + }) + + total = len(result) + + if limit is not None: + result = result[limit * (page - 1):limit * page] + last_page = math.ceil(total / limit) + else: + last_page = 1 + + return { + 'data': result, + 'meta': { + 'last_page': last_page, + 'total': total, + } + } + + +def list_rules(e_uci: EUci) -> list[dict[str]]: + """ + Index all rules + + Args: + - e_uci: euci instance + + Returns: + list of dicts, each dict contains the property "config-name", "description", "enabled", "interface" and "blocks" + """ + rules = list[dict[str]]() + fetch_rules = utils.get_all_by_type(e_uci, 'dpi', 'rule') + + if not fetch_rules: + return rules + + for rule_name in fetch_rules.keys(): + # skipping rules with criteria, must be custom entries + if e_uci.get('dpi', rule_name, 'criteria', default=None) is None: + # load blocklist of applications and protocols + blocklist = __load_blocklist() + # get content of rule + rule = fetch_rules[rule_name] + # prepare the data to append to rules + data_rule = dict[str]() + data_rule['config-name'] = rule_name + data_rule['enabled'] = rule.get('enabled', '1') == '1' + data_rule['device'] = rule.get('device', '*') + # from device, get the interface + interface = utils.get_interface_from_device(e_uci, data_rule['device']) + if interface is not None: + data_rule['interface'] = interface + data_rule['action'] = rule.get('action') + # get the blocked applications/protocols + data_rule['criteria'] = list[dict[str]]() + + # filter by application + application_blocklist = [item for item in blocklist if item['type'] == 'application'] + for application in rule.get('application', []): + found_app = [item for item in application_blocklist if item['name'] == application] + # there's a possibility of not finding the application due to manual edit of the config + if len(found_app) > 0: + data_rule['criteria'].append(found_app[0]) + + # filter by protocol + protocol_blocklist = [item for item in blocklist if item['type'] == 'protocol'] + for protocol in rule.get('protocol', []): + found_protocol = [item for item in protocol_blocklist if item['name'] == protocol] + # there's a possibility of not finding the protocol due to manual edit of the config + if len(found_protocol) > 0: + data_rule['criteria'].append(found_protocol[0]) + + # append rule + rules.append(data_rule) + + return rules + + +def __save_rule_data(e_uci: EUci, config_name: str, enabled: bool, device: str, action: str, applications: list[str], + protocols: list[str]): + e_uci.set('dpi', config_name, 'enabled', enabled) + e_uci.set('dpi', config_name, 'device', device) + e_uci.set('dpi', config_name, 'action', action) + e_uci.set('dpi', config_name, 'application', applications) + e_uci.set('dpi', config_name, 'protocol', protocols) + +def __save_exemption_data(e_uci: EUci, config_name: str, criteria: str, description: str, enabled: bool): + e_uci.set('dpi', config_name, 'enabled', enabled) + e_uci.set('dpi', config_name, 'criteria', criteria) + e_uci.set('dpi', config_name, 'description', description) + +def __toggle_engine(e_uci: EUci): + count_enabled = 0 + for section in e_uci.get_all('dpi'): + if e_uci.get('dpi', section, default="") == "rule" and e_uci.get('dpi', section, 'enabled', default="0") == "1": + count_enabled = count_enabled + 1 + + if count_enabled > 0: + e_uci.set('dpi', 'config', 'enabled', '1') + else: + e_uci.set('dpi', 'config', 'enabled', '0') + +def add_rule(e_uci: EUci, enabled: bool, device: str, action: str, applications: list[str], + protocols: list[str]) -> str: + """ + Store a new rule + + Args: + - e_uci: euci instance + - description: description of the rule + - enabled: enable the rule + - action: apply specific action to rule, can be 'block', 'bulk', 'best_effort', 'video' or 'voice' + - device: device to listen and apply the rule on + - applications: list of applications to block + - protocols: list of protocols to block + + Returns: + config name of the rule created + """ + rule_name = utils.get_random_id() + e_uci.set('dpi', rule_name, 'rule') + __save_rule_data(e_uci, rule_name, enabled, device, action, applications, protocols) + __toggle_engine(e_uci) + e_uci.save('dpi') + return rule_name + + +def delete_rule(e_uci: EUci, config_name: str): + """ + Delete a rule + + Args: + - e_uci: euci instance + - config_name: config name of the rule to delete + """ + e_uci.delete('dpi', config_name) + __toggle_engine(e_uci) + e_uci.save('dpi') + + +def edit_rule(e_uci: EUci, config_name: str, enabled: bool, device: str, action: str, applications: list[str], + protocols: list[str]): + """ + Edit a rule + + Args: + - e_uci: euci instance + - config_name: rule to change + - enabled: enable the rule + - device: device to listen and apply the rule on + - action: apply specific action to rule, can be 'block', 'bulk', 'best_effort', 'video' or 'voice' + - applications: array of applications to block + - protocols: array of protocols to block + + Raises + - ValidationError: if the config name is invalid + """ + if e_uci.get('dpi', config_name, default=None) is None: + raise ValidationError('config-name', 'invalid', config_name) + + __save_rule_data(e_uci, config_name, enabled, device, action, applications, protocols) + __toggle_engine(e_uci) + + e_uci.save('dpi') + +def list_exemptions(e_uci: EUci) -> list[dict[str]]: + """ + Index all global exemptions + + Args: + - e_uci: euci instance + + Returns: + list of dicts, each dict contains the property "config-name", "description", "enabled", "criteria" + """ + exemptions = list[dict[str]]() + fetch_ex = utils.get_all_by_type(e_uci, 'dpi', 'exemption') + + if not fetch_ex: + return exemptions + for ex_name in fetch_ex.keys(): + # get content of exemption + ex = fetch_ex[ex_name] + # prepare the data to append to rules + data_ex = dict[str]() + data_ex['config-name'] = ex_name + data_ex['enabled'] = ex.get('enabled', '1') == '1' + data_ex['criteria'] = ex.get('criteria', '') + data_ex['description'] = ex.get('description', '') + # append exemption + exemptions.append(data_ex) + + return exemptions + + +def add_exemption(e_uci: EUci, criteria: str, description: str, enabled: bool): + """ + Store a new global exemption + + Args: + - e_uci: euci instance + - criteria: exemption criteria, usually it's an IP address + - description: description of the rule + - enabled: enable the exemption + + Returns: + config name of the exemption created + """ + ex_list = utils.get_all_by_type(e_uci, 'dpi', 'exemption') + for ex_name in ex_list: + ex = ex_list[ex_name] + if ex.get('criteria', '') == criteria: + raise ValidationError('criteria', 'criteria_already_exists', criteria) + + ex_name = utils.get_random_id() + e_uci.set('dpi', ex_name, 'exemption') + __save_exemption_data(e_uci, ex_name, criteria, description, enabled) + e_uci.save('dpi') + return ex_name + + +def delete_exemption(e_uci: EUci, config_name: str): + """ + Delete a global exemption + + Args: + - e_uci: euci instance + - config_name: config name of the rule to delete + """ + e_uci.delete('dpi', config_name) + e_uci.save('dpi') + + +def edit_exemption(e_uci: EUci, config_name: str, criteria: str, description: str, enabled: bool): + """ + Edit a global exemption + + Args: + - e_uci: euci instance + - config_name: rule to change + - criteria: exemption criteria, usually it's an IP address + - description: description of the rule + - enabled: enable the exemption + + Raises + - ValidationError: if the config name is invalid + """ + if e_uci.get('dpi', config_name, default=None) is None: + raise ValidationError('config-name', 'invalid', config_name) + + __save_exemption_data(e_uci, config_name, criteria, description, enabled) + e_uci.save('dpi') diff --git a/packages/python3-nethsec/src/nethsec/firewall/__init__.py b/packages/python3-nethsec/src/nethsec/firewall/__init__.py new file mode 100644 index 000000000..4adb67ba5 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/firewall/__init__.py @@ -0,0 +1,2146 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +Firewall utilities +''' +import ipaddress +import json +import os +import subprocess + +from euci import EUci + +from nethsec import utils, objects + +PROTOCOLS = ['tcp', 'udp', 'udplite', 'icmp', 'esp', 'ah', 'sctp'] +TARGETS = ['ACCEPT', 'DROP', 'REJECT', 'NOTRACK'] + +def add_device_to_zone(uci, device, zone): + ''' + Add given device to a firewall zone. + The device is not added if the firewall zone does not exists + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - device -- Device name + - zone -- Firewall zone name + + Returns: + - If the firewall zone exists, the name of the section where the device has been added. + - None, otherwise. + ''' + for section in uci.get("firewall"): + s_type = uci.get("firewall", section) + if s_type == "zone": + zname = uci.get("firewall", section, "name") + if zname == zone: + try: + devices = list(uci.get_all("firewall", section, "device")) + except: + devices = [] + if not device in devices: + devices.append(device) + uci.set("firewall", section, "device", devices) + uci.save("firewall") + return section + + return None + +def add_interface_to_zone(uci, interface, zone): + ''' + Add given interface to a firewall zone. + The interface is not added if the firewall zone does not exists + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - interface -- Interface name + - zone -- Firewall zone name + + Returns: + - If the firewall zone exists, the name of the section where the device has been added. + - None, otherwise. + ''' + for section in uci.get("firewall"): + s_type = uci.get("firewall", section) + if s_type == "zone": + zname = uci.get("firewall", section, "name") + if zname == zone: + try: + networks = list(uci.get_all("firewall", section, "network")) + except: + networks = [] + if not interface in networks: + networks.append(interface) + uci.set("firewall", section, "network", networks) + uci.save("firewall") + return section + + return None + + +def remove_interface_from_zone(uci, interface, zone): + ''' + Remove the given interface from a firewall zone. + The operation always succeed if the zone does not exists + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - interface -- Interface name + - zone -- Firewall zone name + + Returns: + - If the firewall zone exists, the name of the section where the interface has been removed. + - None, otherwise. + ''' + + for z in utils.get_all_by_type(uci, 'firewall', 'zone'): + if uci.get('firewall', z, 'name') == zone: + try: + networks = list(uci.get_all("firewall", z, "network")) + except: + networks = [] + if interface in networks: + networks.remove(interface) + uci.set("firewall", z, "network", networks) + uci.save("firewall") + return z + return None + +def remove_device_from_zone(uci, device, zone): + ''' + Remove the given device from a firewall zone. + The operation always succeed if the zone does not exists + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - device -- Device name + - zone -- Firewall zone name + + Returns: + - If the firewall zone exists, the name of the section where the device has been removed. + - None, otherwise. + ''' + + for z in utils.get_all_by_type(uci, 'firewall', 'zone'): + if uci.get('firewall', z, 'name') == zone: + try: + devices = list(uci.get_all("firewall", z, "device")) + except: + devices = [] + if device in devices: + devices.remove(device) + uci.set("firewall", z, "device", devices) + uci.save("firewall") + return z + return None + +def add_device_to_lan(uci, device): + ''' + Shortuct to add a device to lan zone + + Arguments: + - uci -- EUci pointer + - device -- Device name + + Returns: + - The name of section or None + ''' + return add_device_to_zone(uci, device, 'lan') + +def add_device_to_wan(uci, device): + ''' + Shortuct to add a device to wan zone + + Arguments: + - uci -- EUci pointer + - device -- Device name + + Returns: + - The name of the configuration section or None + ''' + return add_device_to_zone(uci, device, 'wan') + +def add_vpn_interface(uci, name, device, link=""): + ''' + Create a network interface for the given device. + The interface can be used for PBR (Policy Based Routing). + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Interface name + - device -- Device name + - link -- A reference to an existing key in the format / (optional) + + Returns: + - The name of the configuration section or None in case of error + ''' + iname = utils.sanitize(name) + uci.set('network', iname, 'interface') + uci.set('network', iname, 'proto', 'none') + uci.set('network', iname, 'device', device) + uci.set('network', iname, 'ns_tag', ["automated"]) + if link: + uci.set('network', iname, 'ns_link', link) + uci.save('network') + return iname + +def add_trusted_zone(uci, name, networks = [], link = ""): + ''' + Create a trusted zone. The zone will: + - be able to access lan and wan zone + - be accessible from lan zone + + If a zone with the same name already exists, do not recreate it. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Zone name, maximum length is 12 + - network -- A list of interfaces to be added to the zone (optional) + - link -- A reference to an existing key in the format / (optional) + + Returns a tuple: + - The name of the configuration section or None in case of error + - A list of configuration sections or an empy list in case of error + ''' + + if len(name) > 12: + return None, None + + # avoid duplicated zones + zones = utils.get_all_by_type(uci, 'firewall', 'zone') + for z in zones: + if zones[z].get("name", "") == name: + return None, None + forwardings = list() + zname = utils.get_random_id() + name = utils.sanitize(name) + uci.set("firewall", zname, 'zone') + uci.set("firewall", zname, 'name', name) + uci.set("firewall", zname, 'input', 'ACCEPT') + uci.set("firewall", zname, 'output', 'ACCEPT') + uci.set("firewall", zname, 'forward', 'REJECT') + if networks: + uci.set("firewall", zname, 'network', networks) + uci.set("firewall", zname, "ns_tag", ["automated"]) + if link: + uci.set("firewall", zname, "ns_link", link) + + flan = utils.get_random_id() + uci.set("firewall", flan, "forwarding") + uci.set("firewall", flan, "src", name) + uci.set("firewall", flan, "dest", "lan") + uci.set("firewall", flan, "ns_tag", ["automated"]) + if link: + uci.set("firewall", flan, "ns_link", link) + forwardings.append(flan) + + flan = utils.get_random_id() + uci.set("firewall", flan, "forwarding") + uci.set("firewall", flan, "src", "lan") + uci.set("firewall", flan, "dest", name) + uci.set("firewall", flan, "ns_tag", ["automated"]) + if link: + uci.set("firewall", flan, "ns_link", link) + forwardings.append(flan) + + flan = utils.get_random_id() + uci.set("firewall", flan, "forwarding") + uci.set("firewall", flan, "src", name) + uci.set("firewall", flan, "dest", "wan") + uci.set("firewall", flan, "ns_tag", ["automated"]) + if link: + uci.set("firewall", flan, "ns_link", link) + forwardings.append(flan) + uci.save('firewall') + reorder_firewall_config(uci) + + return zname, forwardings + +def add_service(uci, name, port, proto, link = "", tags=None): + ''' + Create an ACCEPT traffic rule for the given service + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Service name + - port -- Service port number as string or array + - proto -- List of service protocols + - link -- A reference to an existing key in the format / (optional) + + Returns: + - The name of the configuration section + ''' + if tags is None: + tags = ['automated'] + rname = utils.get_id(f"allow_{name}") + uci.set("firewall", rname, "rule") + uci.set("firewall", rname, "name", f"Allow-{name}") + uci.set("firewall", rname, "ns_service", "custom") + uci.set("firewall", rname, "src", "wan") + uci.set("firewall", rname, "dest_port", port) + if type(proto) is str: + proto = [proto] + uci.set("firewall", rname, "proto", proto) + uci.set("firewall", rname, "target", "ACCEPT") + uci.set("firewall", rname, "enabled", "1") + uci.set("firewall", rname, "ns_tag", tags) + if link: + uci.set("firewall", rname, "ns_link", link) + uci.save('firewall') + reorder_firewall_config(uci) + + return rname + +def remove_service(uci, name): + ''' + Remove the ACCEPT traffic rule for the given service + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Service name + + Returns: + - The name of the configuration section + ''' + rname = utils.get_id(f"allow_{name}") + uci.delete("firewall", rname) + uci.save('firewall') + reorder_firewall_config(uci) + + return rname + +def disable_service(uci, name): + ''' + Disable the ACCEPT rule traffic for the given service. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Service name + + Returns: + - The name of the configuration section if found, None otherwise + ''' + rname = utils.get_id(f"allow_{name}") + try: + uci.set("firewall", rname, "enabled", "0") + except: + return None + uci.save("firewall") + return rname + +def enable_service(uci, name): + ''' + Disable the ACCEPT rule traffic for the given service + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Service name + + Returns: + - The name of the configuration section if found, None otherwise + ''' + rname = utils.get_id(f"allow_{name}") + try: + uci.set("firewall", rname, "enabled", "0") + except: + return None + uci.save("firewall") + return rname + +def apply(uci): + ''' + Apply firewall configuration: + - commit changes to firewall config + - reload the firewall service + + Arguments: + - uci -- EUci pointer + ''' + uci.commit('firewall') + subprocess.run(["/etc/init.d/firewall", "reload"], check=True) + + +def add_template_forwarding(uci, name, link = ""): + ''' + Create a forwarding from templates database. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Name of the template forwarding from the templates database + - link -- A reference to an existing key in the format / (optional) + + Returns a tuple: + - The name of the configuration section for the forwarding or None in case of error + ''' + + frecord = uci.get_all("templates", name) + fname = utils.get_random_id() + uci.set("firewall", fname, "forwarding") + for section in frecord: + uci.set("firewall", fname, section, frecord[section]) + uci.set("firewall", fname, "ns_tag", ["automated"]) + if link: + uci.set("firewall", fname, "ns_link", link) + uci.save('firewall') + reorder_firewall_config(uci) + + return fname + +def add_template_zone(uci, name, networks = [], link = ""): + ''' + Create a zone from templates database. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Name of the zone from the templates database + - network -- A list of interfaces to be added to the zone (optional) + - link -- A reference to an existing key in the format / (optional) + + Returns a tuple: + - The name of the configuration section for the zone or None in case of error + - A list of configuration section names for the forwardings or None in case of error + ''' + + dzone = uci.get_all("templates", name) + # Search for zones with the same "name" + for zone in utils.get_all_by_type(uci, "firewall", "zone"): + if uci.get("firewall", zone, "name") == dzone["name"]: + return None, None + + forwardings = list() + zname = utils.get_random_id() + forward_list = dzone.pop("forwardings", list()) + uci.set("firewall", zname, "zone") + for section in dzone: + uci.set("firewall", zname, section, dzone[section]) + if len(networks) > 0: + uci.set("firewall", zname, "network", networks) + uci.set("firewall", zname, "ns_tag", ["automated"]) + if link: + uci.set("firewall", zname, "ns_link", link) + + for forward in forward_list: + forwardings.append(add_template_forwarding(uci, forward, link)) + uci.save('firewall') + reorder_firewall_config(uci) + return (zname, forwardings) + +def add_template_service_group(uci, name, src='lan', dest='wan', link=""): + ''' + Create all rules for the given service group + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Name of the service group from the templates database + - src -- Source zone, default is 'lan'. The zone must already exists inside the firewall db + - dest -- Destination zone, default is 'wan'. The zone must already exists inside the firewall db + - link -- A reference to an existing key in the format / (optional) + + Returns: + - A list of configuration section names of each rule, None in case of error + ''' + + group = uci.get_all("templates", name) + services = group.pop("services", list()) + + if not services: + return None + + rules = dict() + sections = list() + + for service in services: + (port, proto, sdesc) = service.split("/",2) + if proto not in rules: + rules[proto] = {"ports": list(), "description": ""} + rules[proto]["ports"].append(port) + rules[proto]["description"] = utils.sanitize(sdesc) + + for proto in rules: + sname = utils.get_random_id() + desc = rules[proto]["description"] + uci.set("firewall", sname, "rule") + uci.set("firewall", sname, "name", f"Allow-{name}-{proto}") + uci.set("firewall", sname, "ns_description", f"{desc} - {proto}") + uci.set("firewall", sname, "src", src) + uci.set("firewall", sname, "dest", dest) + uci.set("firewall", sname, "proto", proto) + uci.set("firewall", sname, "dest_port", ",".join(rules[proto]["ports"])) + uci.set("firewall", sname, "target", "ACCEPT") + uci.set("firewall", sname, "enabled", "1") + uci.set("firewall", sname, "ns_tag", ["automated"]) + if link: + uci.set("firewall", sname, "ns_link", link) + sections.append(sname) + uci.save('firewall') + reorder_firewall_config(uci) + return sections + +def add_template_rule(uci, name, proto="", port="", link="", tags=None): + ''' + Create a rule from templates database. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - name -- Name of the template rule from the templates database + - proto -- A valid UCI protocol (optional) + - port -- A port or comma-separated list of ports (optional) + - link -- A reference to an existing key in the format / (optional) + + Returns: + - The name of the configuration section for the rule or None in case of error + ''' + + if tags is None: + tags = ['automated'] + drule = uci.get_all("templates", name) + rname = utils.get_random_id() + uci.set("firewall", rname, "rule") + for section in drule: + if port: + drule[section] = drule[section].replace("__PORT__", port) + if proto: + drule[section] = drule[section].replace("__PROTO__", proto) + uci.set("firewall", rname, section, drule[section]) + uci.set("firewall", rname, "ns_tag", tags) + if link: + uci.set("firewall", rname, "ns_link", link) + uci.save('firewall') + reorder_firewall_config(uci) + return rname + +def get_all_linked(uci, link): + ''' + Search all database, execpt templates one, for entities with the given link + + Arguments: + - uci -- EUci pointer + - link -- A reference to an existing key in the format / + + Returns: + - A dictionary of all matched sections like + {"db1": ["key1", "key2"], "db2": [...] } + ''' + + ret = dict() + for config in uci.list_configs(): + if config == "templates" or config.endswith(".backup"): + continue + records = utils.get_all_by_option(uci, config, 'ns_link', link, deep = False) + ret[config] = records + + return ret + +def disable_linked_rules(uci, link): + ''' + Disable all rules matching the given link + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - link -- A reference to an existing key in the format / + + Returns: + - A list of disabled sections + ''' + + disabled = list() + linked = get_all_linked(uci, link) + if "firewall" in linked: + for section in linked["firewall"]: + if uci.get("firewall", section) == "rule": + uci.set("firewall", section, "enabled", 0) + disabled.append(section) + + uci.save("firewall") + return disabled + +def delete_linked_sections(uci, link): + ''' + Delete all sections matching the given link. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + - link -- A reference to an existing key in the format / + + Returns: + - A list of deleted sections + ''' + + deleted = list() + linked = get_all_linked(uci, link) + for db in linked: + for section in linked[db]: + uci.delete(db, section) + deleted.append(section) + uci.save(db) + + return deleted + +def is_ipv6_enabled(uci): + ''' + Search the network database for devices and interfaces using IPv6 + + Arguments: + - uci -- EUci pointer + + Returns: + - True if IPv6 is enabled at least on a device or interface, False otherwise + ''' + + for interface in utils.get_all_by_type(uci, 'network', 'interface'): + for option in uci.get_all('network', interface): + if option.startswith("ip6") or option == "dhcpv6" or option == "ipv6": + return True + if uci.get('network', interface, 'proto', default="") in ['6in4', '6to4', '6rd', 'grev6', 'grev6tap', 'vtiv6']: + return True + + for device in utils.get_all_by_type(uci, 'network', 'device'): + try: + if uci.get_all('network', device, 'ipv6') == 1: + return True + except: + pass + return False + +def disable_ipv6_firewall(uci): + ''' + Disable all rules, forwardings, redirects, zones and ipsets for ipv6-only family. + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of disabled sections + ''' + + disabled = list() + for section_type in ["rule", "forwarding", "redirect", "zone", "ipset"]: + for section in utils.get_all_by_type(uci, 'firewall', section_type): + if uci.get("firewall", section, 'family', default="any") == "ipv6": + uci.set("firewall", section, "enabled", "0") + disabled.append(section) + + uci.save("firewall") + return disabled + + +def list_zones(uci) -> dict: + """ + Get all zones from firewall config + + Args: + uci: EUci pointer + + Returns: + dict with all zones + """ + return utils.get_all_by_type(uci, 'firewall', 'zone') + + +def list_zones_no_aliases(uci) -> dict: + """ + Get all zones from firewall config, excluding aliases in network section + + Args: + - uci: EUci pointer + + Returns: + dict with all zones + """ + zones = list_zones(uci) + for name, zone in zones.items(): + result = [] + networks = zone.get("network", []) + for network in networks: + config_network = uci.get("network", network, "device", default="", dtype=str) + if not config_network.startswith("@"): + result.append(network) + + zone["network"] = result + + return zones + + +def list_forwardings(uci) -> dict: + """ + Get all forwardings from firewall config + + Args: + uci: EUci pointer + + Returns: + dict with all forwardings + """ + return utils.get_all_by_type(uci, 'firewall', 'forwarding') + + +def add_forwarding(uci, src: str, dest: str) -> str: + """ + Add forwarding from src to dest. + + Args: + uci: EUci pointer + src: source zone, must be zone name, not config name + dest: destination zone, must be zone name, not config name + + Returns: + name of forwarding config that was added + """ + config_name = utils.get_id(f'{src}2{dest}') + uci.set('firewall', config_name, 'forwarding') + uci.set('firewall', config_name, 'src', src) + uci.set('firewall', config_name, 'dest', dest) + uci.save('firewall') + return config_name + +def get_zone_by_name(uci, name: str) -> str: + """ + Get zone config name by zone name. + + Args: + uci: EUci pointer + name: zone name + + Returns: + tuple of zone config name and zone config if zone with name name exists, (None, None) otherwise + """ + zones = utils.get_all_by_type(uci, 'firewall', 'zone') + for z in zones: + if uci.get('firewall', z, 'name', default='') == name: + return (z, zones[z]) + return (None, None) + +def get_rule_by_name(uci, name: str, tag = "") -> str: + """ + Get rule config name and rule data by rule name, optionally filtered by tag. + Assume there is only one rule with the same name. + + Args: + uci: EUci pointer + name: rule name + tag: optional tag to filter rules + + Returns: + tuple of rule config name and rule config if rule with name name exists, (None, None) otherwise + """ + rules = utils.get_all_by_type(uci, 'firewall', 'rule') + for r in rules: + if uci.get('firewall', r, 'name', default='') == name: + if not tag or tag in uci.get('firewall', r, 'ns_tag', default=[]): + return (r, rules[r]) + return (None, None) + + +def zone_exists(u, zone_name): + """ + Check if a zone with name zone_name already exists + + Args: + u: EUci pointer + zone_name: zone name to check + + Returns: + true if a zone with name zone_name already exists, false otherwise + """ + try: + for h in utils.get_all_by_type(u, 'firewall', 'zone'): + if u.get('firewall', h, 'name', default='') == zone_name: + return True + except: + return False + + return False + + +def add_zone(uci, name: str, input: str, forward: str, traffic_to_wan: bool = False, forwards_to: list[str] = None, + forwards_from: list[str] = None, log: bool = False) -> {str, set[str]}: + """ + Add zone to firewall config. + + Args: + uci: EUci pointer + name: name of zone + input: rule for input traffic, must be one of 'ACCEPT', 'REJECT', 'DROP' + forward: rule for forward traffic, must be one of 'ACCEPT', 'REJECT', 'DROP' + traffic_to_wan: if True, add forwarding from zone to wan + forwards_to: list of zones to forward traffic to + forwards_from: list of zones to forward traffic from + log: if True, log blocked traffic destined to this zone + + Returns: + tuple of zone config name and set of added forwarding configs + """ + + # check if a zone with the same name already exists + if zone_exists(uci, name): + return utils.validation_error("name", "zone_already_exists", name) + + zone_config_name = utils.get_id(name) + uci.set('firewall', zone_config_name, 'zone') + uci.set('firewall', zone_config_name, 'name', name) + uci.set('firewall', zone_config_name, 'input', input) + uci.set('firewall', zone_config_name, 'forward', forward) + uci.set('firewall', zone_config_name, 'output', 'ACCEPT') + if log: + uci.set('firewall', zone_config_name, 'log', '1') + log_opts = get_default_logging_options(uci) + uci.set('firewall', zone_config_name, 'log_limit', log_opts['zone_log_limit']) + else: + uci.set('firewall', zone_config_name, 'log', '0') + try: + uci.delete('firewall', zone_config_name, 'log_limit') + except: + pass + + + forwardings_added = set() + + if traffic_to_wan: + forwardings_added.add(add_forwarding(uci, name, 'wan')) + + if forwards_to is not None: + for forward_to in forwards_to: + forwardings_added.add(add_forwarding(uci, name, forward_to)) + + if forwards_from is not None: + for forward_from in forwards_from: + forwardings_added.add(add_forwarding(uci, forward_from, name)) + + uci.save('firewall') + reorder_firewall_config(uci) + return zone_config_name, forwardings_added + + +def edit_zone(uci, name: str, input: str, forward: str, traffic_to_wan: bool = False, forwards_to: list[str] = None, + forwards_from: list[str] = None, log: bool = False) -> {str, set[str]}: + """ + Edit an existing zone. + + Args: + uci: EUci pointer + name: name of zone to edit + input: rule for input traffic, must be one of 'ACCEPT', 'REJECT', 'DROP' + forward: rule for forward traffic, must be one of 'ACCEPT', 'REJECT', 'DROP' + traffic_to_wan: if True, add forwarding from zone to wan + forwards_to: list of zones to forward traffic to + forwards_from: list of zones to forward traffic from + log: if True, log blocked traffic destined to this zone + + Returns: + tuple of zone config name and set of updated forwarding configs + """ + (zone_config_name, zone) = get_zone_by_name(uci, name) + if zone is None or zone_config_name is None: + return utils.validation_error("name", "zone_does_not_exists", name) + + uci.set('firewall', zone_config_name, 'input', input) + uci.set('firewall', zone_config_name, 'forward', forward) + uci.set('firewall', zone_config_name, 'output', 'ACCEPT') + if log: + uci.set('firewall', zone_config_name, 'log', '1') + if uci.get('firewall', zone_config_name, 'log_limit', default=None) is None: + # set default log limit, do not overwrite existing value + log_opts = get_default_logging_options(uci) + uci.set('firewall', zone_config_name, 'log_limit', log_opts['zone_log_limit']) + else: + uci.set('firewall', zone_config_name, 'log', '0') + try: + uci.delete('firewall', zone_config_name, 'log_limit') + except: + pass + + # delete old forwardings + + forwardings = list_forwardings(uci) + to_delete_forwardings = set() + for forwarding in forwardings: + if forwardings[forwarding]['src'] == name: + to_delete_forwardings.add(forwarding) + if forwardings[forwarding]['dest'] == name: + to_delete_forwardings.add(forwarding) + + for to_delete_forwarding in to_delete_forwardings: + uci.delete('firewall', to_delete_forwarding) + + # create updated forwardings + + forwardings_added = set() + + if traffic_to_wan: + forwardings_added.add(add_forwarding(uci, name, 'wan')) + + if forwards_to is not None: + for forward_to in forwards_to: + forwardings_added.add(add_forwarding(uci, name, forward_to)) + + if forwards_from is not None: + for forward_from in forwards_from: + forwardings_added.add(add_forwarding(uci, forward_from, name)) + + uci.save('firewall') + return zone_config_name, forwardings_added + + +def delete_zone(uci, zone_config_name: str) -> {str, set[str]}: + """ + Delete zone and all forwardings that are connected to it. + + Args: + uci: EUci pointer + zone_config_name: name of zone config to delete + + Returns: + tuple of zone config name and set of deleted forwarding configs + + Raises: + ValueError: if zone_config_name is not a valid zone config name + """ + if zone_config_name not in list_zones(uci): + raise ValueError + zone_name = list_zones(uci)[zone_config_name]['name'] + forwardings = list_forwardings(uci) + to_delete_forwardings = set() + for forwarding in forwardings: + if forwardings[forwarding]['src'] == zone_name: + to_delete_forwardings.add(forwarding) + if forwardings[forwarding]['dest'] == zone_name: + to_delete_forwardings.add(forwarding) + + for to_delete_forwarding in to_delete_forwardings: + uci.delete('firewall', to_delete_forwarding) + uci.delete('firewall', zone_config_name) + uci.save('firewall') + reorder_firewall_config(uci) + return zone_config_name, to_delete_forwardings + +def add_default_ipv6_rules(uci): + """ + Add default ipv6 rules to firewall config, if they don't exist already. + + Args: + uci: EUci pointer + + Returns: + list of added rule config names + """ + ret = list() + rules = {"ip6_dhcp" : "Allow-DHCPv6", "ip6_mld" : "Allow-MLD", "ip6_icmp" : "Allow-ICMPv6-Input", "ip6_icmp_forward" : "Allow-ICMPv6-Forward"} + for r in rules: + (rule_name, rule) = get_rule_by_name(uci, rules[r], tag="automated") + if rule_name is None: + ret.append(add_template_rule(uci, r)) + return ret + +def delete_rule(uci, id: str) -> str: + """ + Delete rule from firewall config. + + Args: + uci: EUci pointer + id: name of rule config to delete + + Returns: + name of rule config that was deleted + + Raises: + ValueError: if id is not a valid rule config name + """ + if id not in list_rule_ids(uci): + raise utils.ValidationError('id', 'rule_not_found', id) + uci.delete('firewall', id) + uci.save('firewall') + reorder_firewall_config(uci) + return id + +def disable_rule(uci, id: str) -> str: + """ + Disable rule from firewall config. + + Args: + uci: EUci pointer + id: name of rule config to disable + + Returns: + name of rule config that was disabled + + Raises: + ValueError: if id is not a valid rule config name + """ + if id not in list_rule_ids(uci): + raise utils.ValidationError('id', 'rule_not_found', id) + uci.set('firewall', id, 'enabled', '0') + uci.save('firewall') + return id + +def enable_rule(uci, id: str) -> str: + """ + Enable rule from firewall config. + + Args: + uci: EUci pointer + id: name of rule config to enable + + Returns: + name of rule config that was enabled + + Raises: + ValueError: if id is not a valid rule config name + """ + if id not in list_rule_ids(uci): + raise utils.ValidationError('id', 'rule_not_found', id) + uci.set('firewall', id, 'enabled', '1') + uci.save('firewall') + return id + +def list_rule_ids (uci) -> list[str]: + """ + Get all rule ids from firewall config + + Args: + uci: EUci pointer + + Returns: + list of all rule ids + """ + return list(utils.get_all_by_type(uci, 'firewall', 'rule').keys()) + +def order_rules(uci, rule_type: str, order: list[str]) -> list[str]: + """ + Orders firewall rules, moves everything else but rules to the end of the list. + + Args: + e_uci: euci instance + rule_type: type of rule to order, must be 'input', 'output' or 'forward' + rules: which order to put rules + + Returns: + list of ordered rules entries + + Raises: + ValidationError: if a rule is not present in /etc/config/firewall + """ + rules = [] + if not rule_type in ['input', 'output', 'forward']: + raise utils.ValidationError('rule_type', 'invalid_rule_type', rule_type) + + (defaults, zones, forwardings, other_rules, forward_rules, output_rules, input_rules) = split_firewall_config(uci) + + if rule_type == 'input': + rules = input_rules + elif rule_type == 'output': + rules = output_rules + elif rule_type == 'forward': + rules = forward_rules + + for r in rules: + if r not in order: + raise utils.ValidationError('order', 'invalid_order', r) + + if rule_type == 'input': + final_order = defaults + zones + forwardings + other_rules + forward_rules + output_rules + order + elif rule_type == 'output': + final_order = defaults + zones + forwardings + other_rules + forward_rules + order + input_rules + elif rule_type == 'forward': + final_order = defaults + zones + forwardings + other_rules + order + output_rules + input_rules + + # enforce new order + index = 0 + for section in final_order: + subprocess.run(["uci", "-c", uci.confdir(), "-t", uci.savedir(), "reorder", f"firewall.{section}={index}"]) + index = index + 1 + uci.save('firewall') + + return order + +def resolve_address(uci, address: str) -> str: + """ + Resolve address to a more human-redeable name. + + Args: + uci: EUci pointer + address: address to resolve + + Returns: + resolved address as a dict with keys value, label and type + """ + for section in uci.get_all("dhcp"): + if uci.get("dhcp", section, "ip", default='') == address: + return {"value": address, "label": uci.get("dhcp", section, "name", default=address), "type": uci.get("dhcp", section)} + + for section in uci.get_all("network"): + if uci.get("network", section, "ipaddr", default='') == address: + return {"value": address, "label": section, "type": uci.get("network", section)} + if uci.get("network", section, "ip6addr", default='') == address: + return {"value": address, "label": section, "type": uci.get("network", section)} + + return {"value": address, "label": None, "type": None} + +def enrich_rule(uci, rule: dict) -> dict: + """ + Enrich rule with more human-readable data and missing fields + + Args: + uci: EUci pointer + rule: rule to enrich + + Returns: + enriched rule + """ + if 'automated' in rule.get('ns_tag', []): + rule['system_rule'] = True + else: + rule['system_rule'] = False + if not 'src_ip' in rule: + rule['src_ip'] = [] + else: + if type(rule['src_ip'] ) == tuple: + rule['src_ip'] = list(rule['src_ip']) + elif type(rule['src_ip'] ) == str: + rule['src_ip'] = [rule['src_ip']] + if not 'dest_ip' in rule: + rule['dest_ip'] = [] + else: + if type(rule['dest_ip'] ) == tuple: + rule['dest_ip'] = list(rule['dest_ip']) + elif type(rule['dest_ip'] ) == str: + rule['dest_ip'] = [rule['dest_ip']] + # try to gather info on src_ip and dest_ip + for key in ['src_ip', 'dest_ip']: + tmp = [] + for ip in rule.get(key, []): + if ip: + tmp.append(resolve_address(uci, ip)) + rule[key] = tmp + proto = rule.get('proto', "") + if not proto: + rule['proto'] = ['udp', 'tcp'] + else: + if type(proto) == tuple: + rule['proto'] = list(proto) + elif type(proto) == str: + rule['proto'] = [proto] + dest_port = rule.get('dest_port', "") + if not dest_port: + rule['dest_port'] = [] + else: + if type(dest_port) != list: + rule['dest_port'] = dest_port.split(" ") + if not 'ns_service' in rule: + if rule.get('proto') != None or rule.get('dest_port') != None: + rule['ns_service'] = 'custom' + else: + rule['ns_service'] = '' + if not 'ns_tag' in rule: + rule['ns_tag'] = [] + rule['log'] = True if rule.get('log', '0') == '1' else False + rule['enabled'] = True if rule.get('enabled', '1') == '1' else False + return rule + +def is_forward_rule(rule: dict) -> bool: + """ + Check if rule is a forward rule + + Args: + rule: rule to check + + Returns: + True if rule is a forward rule, False otherwise + """ + if rule.get('dest') and rule.get('src'): + return True + return False + +def is_input_rule(rule: dict) -> bool: + """ + Check if rule is an input rule + + Args: + rule: rule to check + + Returns: + True if rule is an input rule, False otherwise + """ + if not rule.get('dest') and rule.get('src'): + return True + return False + +def is_output_rule(rule: dict) -> bool: + """ + Check if rule is an output rule + + Args: + rule: rule to check + + Returns: + True if rule is an output rule, False otherwise + """ + if rule.get('dest') and not rule.get('src'): + return True + return False + +def is_zone(uci, name:str) -> bool: + """ + Check if name is a zone or any zone ('*') + + Args: + name: zone to check + + Returns: + True if name is a zone, False otherwise + """ + + # True zone if name is 'any' zone name -> '*' + if name == '*': + return True + + zones = list_zones(uci) + for value in zones.values(): + if 'name' in value and value['name'] == name: + return True + return False + +def list_rules(uci, rule_type = None) -> list: + """ + Get all rules from firewall config + + Args: + uci: EUci pointer + rule_type: optional rule type to filter, must be one of 'input', 'output' or 'forward' + + Returns: + a list of all rules + """ + rules = [] + i = 0 + for section in uci.get_all("firewall"): + if uci.get('firewall', section) == 'rule': + rule = uci.get_all('firewall', section) + rule['id'] = section + rule['index'] = i + if not rule_type: + rules.append(enrich_rule(uci, rule)) + else: + if rule_type =='forward' and is_forward_rule(rule): + rules.append(enrich_rule(uci, rule)) + rule['active_zone'] = is_zone(uci, rule.get('src','*')) and is_zone(uci, rule.get('dest','*')) + elif rule_type =='input' and is_input_rule(rule): + rules.append(enrich_rule(uci, rule)) + rule['active_zone'] = is_zone(uci, rule.get('src','*')) + elif rule_type =='output' and is_output_rule(rule): + rules.append(enrich_rule(uci, rule)) + rule['active_zone'] = is_zone(uci, rule.get('dest','*')) + i += 1 + return rules + +def list_forward_rules(uci) -> list: + """ + Get all forward rules from firewall config + + Args: + uci: EUci pointer + + Returns: + a list of all forward rules + """ + return list_rules(uci, 'forward') + + +def list_output_rules(uci) -> list: + """ + Get all output rules from firewall config + + Args: + uci: EUci pointer + + Returns: + a list of all output rules + """ + return list_rules(uci, 'output') + + +def list_input_rules(uci) -> list: + """ + Get all input rules from firewall config + + Args: + uci: EUci pointer + + Returns: + a list of all input rules + """ + return list_rules(uci, 'input') + + +def list_service_suggestions(): + """ + Get all services from /etc/services + + Returns: + a list of all services, each service is a dict with keys id, port, proto + """ + services = {} + if os.path.isfile('/etc/services'): + with open('/etc/services', 'r') as fp: + for line in fp: + if line.startswith('#'): + continue + try: + tmp = line.split() + name = tmp[0] + (port, proto) = tmp[1].split('/') + except Exception as e: + continue + if port not in services: + services[port] = {"id": name, "proto": [proto]} + else: + services[port]['proto'].append(proto) + else: + return [] + ret = [] + for port in services: + service = services[port] + service['port'] = int(port) + ret.append(service) + return ret + +def list_active_leases(): + """ + Get all active leases from /tmp/dhcp.leases + + Returns: + a list of all active leases, each lease is a dict with keys value, label, type + """ + ret = [] + if os.path.isfile("/tmp/dhcp.leases"): + with open("/tmp/dhcp.leases", "r") as fp: + for line in fp.readlines(): + line = line.strip() + if not line: + continue + tmp = line.split(" ") + if tmp[3] != "*": + ret.append({"value": tmp[2], "label": tmp[3], 'type': 'lease'}) + return ret + +def list_host_suggestions(uci): + """ + Get all hosts from dhcp and network config + + Args: + uci: EUci pointer + + Returns: + a list of all hosts, each host is a dict with keys value, label, type + """ + ret = [] + for section in uci.get_all("dhcp"): + if uci.get("dhcp", section, "ip", default=None): + ret.append({"value": uci.get("dhcp", section, "ip"), "label": uci.get("dhcp", section, "name"), "type": uci.get("dhcp", section)}) + for section in uci.get_all("network"): + ip = uci.get("network", section, "ipaddr", default=None) + if ip and ip != '127.0.0.1': + ret.append({"value": uci.get("network", section, "ipaddr"), "label": section, "type": "network"}) + if uci.get("network", section, "ip6addr", default=None): + ret.append({"value": uci.get("network", section, "ip6addr"), "label": section, "type": "network"}) + ret = ret + list_active_leases() + return ret + +def list_object_suggestions(uci, expand = False): + """ + Get all objects from objects config + + Args: + uci: EUci pointer + expand: if True, expand object details + + Returns: + a list of all objects, each object is a dict with keys value, label, type + """ + return objects.list_all_objects(uci, expand) + +def validate_address_format(address: str) -> bool: + """ + Validate address format. + Valid formats are: + - ip address + - ip range like 192.168.100.1-192.168.100.10 + - ip cidr + + Args: + address: address to validate + + Returns: + True if address is valid, False otherwise + """ + if not address: + return True + try: + if '/' in address: + (ip, cidr) = address.split('/') + ipaddress.ip_address(ip) + cidr = int(cidr) + if cidr < 0 or cidr > 32: + return False + elif '-' in address: + (start, end) = address.split('-') + start = ipaddress.ip_address(start) + end = ipaddress.ip_address(end) + if start > end: + return False + else: + ipaddress.ip_address(address) + except: + return False + return True + +def validate_port_format(port: str) -> bool: + """ + Validate port format. + + Args: + port: port to validate + + Returns: + True if port is valid, False otherwise + """ + if not port: + return True + try: + if '-' in port: + (start, end) = port.split('-') + start = int(start) + end = int(end) + if start < 0 or start > 65535: + return False + if end < 0 or end > 65535: + return False + if start > end: + return False + elif ',' in port: + for p in port.split(','): + p = int(p) + if p < 0 or p > 65535: + return False + else: + port = int(port) + if port < 0 or port > 65535: + return False + except: + return False + return True + +def validate_rule(uci, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, ns_src: str, ns_dst: str): + """ + Validate rule. + + Args: + src: source zone, must be zone name, not config name + src_ip: a list of source ip + dest: destination zone, must be zone name, not config name + dest_ip: a list of destination ip + proto: protocol, must be a list of protocols in "tcp", "udp", "udplite", "icmp", "esp", "ah", "sctp" + dest_port: a list of destination ports, each element cna be be a port number, a comma-separated list of port numbers or a range with `-` (eg. 80-90) + target: target, must be one of 'ACCEPT', 'REJECT', 'DROP', 'NOTRACK' + service: service name + ns_src: an object in the form `/` + ns_dst: an object in the form `/` + + Raises: + ValidationError: if rule is invalid + """ + if ns_src: + if not objects.object_exists(uci, ns_src): + raise utils.ValidationError('ns_src', 'object_not_found', ns_src) + else: # check source only if not using objects + for s in src_ip: + if not validate_address_format(s): + raise utils.ValidationError('src_ip', 'invalid_format', s) + if ns_dst: + if not objects.object_exists(uci, ns_dst): + raise utils.ValidationError('ns_dst', 'object_not_found', ns_dst) + else: # check destiation only if not using objects + for d in dest_ip: + if not validate_address_format(d): + raise utils.ValidationError('dest_ip', 'invalid_format', d) + if ns_src and ns_dst and objects.is_domain_set(uci, ns_src) and objects.is_domain_set(uci, ns_dst): + raise utils.ValidationError('ns_dst', 'domain_set_conflict', ns_dst) + if target not in TARGETS: + raise utils.ValidationError('target', 'invalid_target', target) + if service and service != '*': + if service == 'custom': + for p in proto: + if p not in PROTOCOLS: + raise utils.ValidationError('proto', 'invalid_proto', p) + for port in dest_port: + if not validate_port_format(port): + raise utils.ValidationError('dest_port', 'invalid_port', port) + else: + services = list(map(lambda x: x['id'], list_service_suggestions())) + if services and service not in services: + raise utils.ValidationError('ns_service', 'invalid_service', service) + +def get_service_by_name(name: str) -> dict: + """ + Get service by name. + + Args: + name: service name + + Returns: + service dict if service with name name exists, None otherwise + """ + for service in list_service_suggestions(): + if service['id'] == name: + return service + return None + +def setup_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, + enabled: bool = True, log: bool = False, tag = [], ns_src: str = None, ns_dst: str = None, ns_link: str = None) -> None: + """ + Set up a rule in the firewall config. + + Args: + uci: EUci pointer + id: id of the rule + name: name of the rule + src: source zone, must be zone name, not config name + src_ip: a list of source IP addresses + dest: destination zone, must be zone name, not config name + dest_ip: a list of destination IP addresses + proto: protocol, must be a list of protocols in "tcp", "udp", "udplite", "icmp", "esp", "ah", "sctp" + dest_port: a list of destination ports, each element can be a port number, a comma-separated list of port numbers, or a range with `-` (e.g., 80-90) + target: target, must be one of 'ACCEPT', 'REJECT', 'DROP', 'NOTRACK' + service: service name + enabled: if True, rule is enabled; if False, rule is disabled + log: if True, log traffic + tag: list of optional tags + ns_src: an object in the form `/` + ns_dst: an object in the form `/` + ns_link: a string in the form `/` that is used to link this rule to another entity + """ + uci.set('firewall', id, 'name', name) + uci.set('firewall', id, 'src', src) + uci.set('firewall', id, 'src_ip', src_ip) + uci.set('firewall', id, 'dest', dest) + uci.set('firewall', id, 'dest_ip', dest_ip) + + uci.set('firewall', id, 'target', target) + if service and service != '*': + if service == 'custom': + uci.set('firewall', id, 'ns_service', 'custom') + uci.set('firewall', id, 'proto', proto) + uci.set('firewall', id, 'dest_port', " ".join(dest_port)) + else: + uci.set('firewall', id, 'ns_service', service) + service = get_service_by_name(service) + uci.set('firewall', id, 'proto', service['proto']) + uci.set('firewall', id, 'dest_port', service['port']) + else: + try: + uci.set('firewall', id, 'ns_service', '*') + uci.set('firewall', id, 'proto', 'all') + uci.delete('firewall', id, 'dest_port') + except: + pass + + uci.set('firewall', id, 'enabled', '1' if enabled else '0') + if log: + uci.set('firewall', id, 'log', '1') + if uci.get('firewall', id, 'log_limit', default=None) is None: + # set default log limit, do not overwrite existing value + log_opts = get_default_logging_options(uci) + uci.set('firewall', id, 'log_limit', log_opts['rule_log_limit']) + else: + uci.set('firewall', id, 'log', '0') + try: + uci.delete('firewall', id, 'log_limit') + except: + pass + uci.set('firewall', id, 'ns_tag', tag) + if ns_src: + uci.set('firewall', id, 'ns_src', ns_src) + else: + try: + uci.delete('firewall', id, 'ns_src') + except: + pass + if ns_dst: + uci.set('firewall', id, 'ns_dst', ns_dst) + else: + try: + uci.delete('firewall', id, 'ns_dst') + except: + pass + if ns_link: + uci.set('firewall', id, 'ns_link', ns_link) + else: + try: + uci.delete('firewall', id, 'ns_link') + except: + pass + uci.save('firewall') + +def split_firewall_config(uci): + """ + Split firewall config into sections. + + Args: + uci: EUci pointer + + Returns: + tuple of lists of sections, in the following order: defaults, zones, forwardings, other_rules, forward_rules, output_rules, input_rules + """ + forward_rules = [] + output_rules = [] + input_rules = [] + other_rules = [] + zones = [] + forwardings = [] + defaults = [] + for section in uci.get_all("firewall"): + record = uci.get_all("firewall", section) + if uci.get('firewall', section) == 'rule': + if is_forward_rule(record): + forward_rules.append(section) + elif is_input_rule(record): + input_rules.append(section) + elif is_output_rule(record): + output_rules.append(section) + else: + other_rules.append(section) + elif uci.get('firewall', section) == 'zone': + zones.append(section) + elif uci.get('firewall', section) == 'forwarding': + forwardings.append(section) + else: + defaults.append(section) + + return (defaults, zones, forwardings, other_rules, forward_rules, output_rules, input_rules) + + +def reorder_firewall_config(uci): + """ + Reorder firewall config, moving all rules at the bottom. + The order in the file will be: + - defaults and includes + - zones + - forwardings + - forward rules + - output rules + - input rules + + Args: + uci: EUci pointer + """ + (defaults, zones, forwardings, other_rules, forward_rules, output_rules, input_rules) = split_firewall_config(uci) + + order = defaults + zones + forwardings + other_rules + forward_rules + output_rules + input_rules + index = 0 + for section in order: + subprocess.run(["uci", "-t", uci.savedir(), "-c", uci.confdir(), "reorder", f"firewall.{section}={index}"]) + index = index + 1 + uci.save('firewall') + +def add_rule(uci, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, + enabled: bool = True, log: bool = False, tag = [], add_to_top: bool = False, ns_src: str = None, ns_dst: str = None, ns_link: str = None) -> str: + """ + Add rule to firewall config. + + Args: + uci: EUci pointer + id: id of rule to edit + name: name of rule + src: source zone, must be zone name, not config name + src_ip: a list of source ip + dest: destination zone, must be zone name, not config name + dest_ip: a list of destination ip + proto: protocol, must be a list of protocols in "tcp", "udp", "udplite", "icmp", "esp", "ah", "sctp" + dest_port: a list of destination ports, each element cna be be a port number, a comma-separated list of port numbers or a range with `-` (eg. 80-90) + target: target, must be one of 'ACCEPT', 'REJECT', 'DROP', 'NOTRACK' + service: service name + enabled: if True, rule is enabled, if False, rule is disabled + log: if True, log traffic + tag: list of optional tags + add_to_top: if True, add rule to the top of the list, otherwise add to the bottom + ns_src: an object in the form `/` + ns_dst: an object in the form `/` + ns_link: a string in the form `/` that is used to link this rule to another entity + + Returns: + name of rule config that was added + """ + validate_rule(uci, src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) + rule = utils.get_random_id() + uci.set('firewall', rule, 'rule') + setup_rule(uci, rule, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag, ns_src, ns_dst, ns_link) + reorder_firewall_config(uci) + update_firewall_rules(uci) # expand objects and save + + if add_to_top: + rule_type = uci.get_all('firewall', rule) + if is_forward_rule(rule_type): + ids = list(map(lambda x: x['id'], list_forward_rules(uci))) + order_rules(uci, 'forward', [rule] + ids[:-1] ) + elif is_input_rule(rule_type): + ids = list(map(lambda x: x['id'], list_input_rules(uci))) + order_rules(uci, 'input', [rule] + ids[:-1] ) + elif is_output_rule(rule_type): + ids = list(map(lambda x: x['id'], list_output_rules(uci))) + order_rules(uci, 'output', [rule] + ids[:-1] ) + return rule + +def edit_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, + enabled: bool = True, log: bool = False, tag = [], ns_src: str = None, ns_dst: str = None, ns_link: str = None) -> str: + """ + Edit rule in firewall config. + + Args: + uci: EUci pointer + id: id of rule to edit + name: name of rule + src: source zone, must be zone name, not config name + src_ip: a list of source ip + dest: destination zone, must be zone name, not config name + dest_ip: a list of destination ip + proto: protocol, must be a list of protocols in "tcp", "udp", "udplite", "icmp", "esp", "ah", "sctp" + dest_port: a list of destination ports, each element cna be be a port number, a comma-separated list of port numbers or a range with `-` (eg. 80-90) + target: target, must be one of 'ACCEPT', 'REJECT', 'DROP', 'NOTRACK' + service: service name + enabled: if True, rule is enabled, if False, rule is disabled + log: if True, log traffic + tag: list of optional tags + ns_src: an object in the form `/` + ns_dst: an object in the form `/` + ns_link: a string in the form `/` that is used to link this rule to another entity + + Returns: + name of rule config that was edited + """ + if not uci.get('firewall', id, default=None): + raise utils.ValidationError("id", "rule_does_not_exists", id) + validate_rule(uci, src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) + setup_rule(uci, id, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag, ns_src, ns_dst, ns_link) + update_firewall_rules(uci) # expand objects and save + return id + +def list_nat_rules(uci) -> list: + """ + Get all nat rules from firewall config + + Args: + uci: EUci pointer + + Returns: + a list of all nat rules + """ + rules = [] + for section in uci.get_all("firewall"): + if uci.get('firewall', section) == 'nat': + rule = uci.get_all('firewall', section) + rule['id'] = section + rule.pop('proto', None) + rules.append(rule) + return rules + +def add_nat_rule(uci, name: str, target: str, src: str = '*', src_ip: str = '', dest_ip: str = '', snat_ip: str = '', device: str = '') -> str: + """ + Add nat rule to firewall config. + + Args: + uci: EUci pointer + name: name of rule + target: target, must be one of 'SNAT', 'DNAT' + src: source zone, must be zone name, not config name + src_ip: source ip + dest_ip: destination ip + snat_ip: snat ip + device: add nat rule just for specific device + + Returns: + name of rule config that was added + """ + if len(name) > 120: + raise utils.ValidationError('name', 'name_too_long', name) + if target not in ["ACCEPT", "MASQUERADE", "SNAT"]: + raise utils.ValidationError('target', 'invalid_target', target) + rule = utils.get_random_id() + uci.set('firewall', rule, 'nat') + uci.set('firewall', rule, 'name', name) + uci.set('firewall', rule, 'target', target) + uci.set('firewall', rule, 'src', src) + uci.set('firewall', rule, 'src_ip', src_ip) + uci.set('firewall', rule, 'dest_ip', dest_ip) + uci.set('firewall', rule, 'snat_ip', snat_ip) + uci.set('firewall', rule, 'proto', ["all"]) + uci.set('firewall', rule, 'device', device) + uci.save('firewall') + return rule + +def edit_nat_rule(uci, id: str, name: str, target: str, src: str = '*', src_ip: str = '', dest_ip: str = '', snat_ip: str = '', device: str = '') -> str: + """ + Edit nat rule in firewall config. + + Args: + uci: EUci pointer + id: id of rule to edit + name: name of rule + target: target, must be one of 'SNAT', 'DNAT' + src: source zone, must be zone name, not config name + src_ip: source ip + dest_ip: destination ip + snat_ip: snat ip + src_device: source device + device: add nat rule just for specific device + + Returns: + name of rule config that was edited + """ + if not uci.get('firewall', id, default=None): + raise utils.ValidationError("id", "rule_does_not_exists", id) + if len(name) > 120: + raise utils.ValidationError('name', 'name_too_long', name) + if target not in ["ACCEPT", "MASQUERADE", "SNAT"]: + raise utils.ValidationError('target', 'invalid_target', target) + uci.set('firewall', id, 'name', name) + uci.set('firewall', id, 'target', target) + uci.set('firewall', id, 'src', src) + uci.set('firewall', id, 'src_ip', src_ip) + uci.set('firewall', id, 'dest_ip', dest_ip) + uci.set('firewall', id, 'snat_ip', snat_ip) + uci.set('firewall', id, 'device', device) + uci.save('firewall') + return id + +def delete_nat_rule(uci, id: str) -> str: + """ + Delete nat rule from firewall config. + + Args: + uci: EUci pointer + id: id of rule to delete + + Returns: + name of rule config that was deleted + """ + if not uci.get('firewall', id, default=None): + raise utils.ValidationError("id", "rule_does_not_exists", id) + uci.delete('firewall', id) + uci.save('firewall') + return id + +def list_netmap_rules(uci) -> list: + """ + Get all netmap rules from netmap config + + Args: + uci: EUci pointer + + Returns: + a list of all netmap rules + """ + rules = [] + for section in uci.get_all("netmap"): + if uci.get('netmap', section) == 'rule': + rule = uci.get_all('netmap', section) + rule['id'] = section + rules.append(rule) + return rules + +def validate_netmap_rule(name: str, src: str, dest: str, map_from: str, map_to: str): + if len(name) > 120: + raise utils.ValidationError('name', 'name_too_long', name) + if src and dest: + raise utils.ValidationError('src', 'src_or_dest', src) + try: + ipaddress.ip_network(map_from) + except: + raise utils.ValidationError('map_from', 'map_from_invalid_format', map_from) + try: + ipaddress.ip_network(map_to) + except: + raise utils.ValidationError('map_to', 'map_to_invalid_format', map_to) + if src: + try: + ipaddress.ip_network(src) + except: + raise utils.ValidationError('src', 'src_invalid_format', src) + if dest: + try: + ipaddress.ip_network(dest) + except: + raise utils.ValidationError('dest', 'dest_invalid_format', dest) + + +def add_netmap_rule(uci, name: str, src: str, dest: str, device_in: list[str], device_out: list[str], map_from: str, map_to: str) -> str: + """ + Add netmap rule to netmap config. + + Args: + uci: EUci pointer + name: name of rule + src: source zone, must be zone name, not config name + dest: destination zone, must be zone name, not config name + device_in: list of incoming network interfaces + device_out: list of outgoing network interfaces + map_from: source network address + map_to: destination network address + + Returns: + name of rule config that was added + """ + validate_netmap_rule(name, src, dest, map_from, map_to) + rule = utils.get_random_id() + uci.set('netmap', rule, 'rule') + uci.set('netmap', rule, 'name', name) + uci.set('netmap', rule, 'src', src) + uci.set('netmap', rule, 'dest', dest) + uci.set('netmap', rule, 'device_in', device_in) + uci.set('netmap', rule, 'device_out', device_out) + uci.set('netmap', rule, 'map_from', map_from) + uci.set('netmap', rule, 'map_to', map_to) + uci.save('netmap') + return rule + +def edit_netmap_rule(uci, id: str, name: str, src: str, dest: str, device_in: list[str], device_out: list[str], map_from: str, map_to: str) -> str: + """ + Edit netmap rule in netmap config. + + Args: + uci: EUci pointer + id: id of rule to edit + name: name of rule + src: source zone, must be zone name, not config name + dest: destination zone, must be zone name, not config name + device_in: list of incoming network interfaces + device_out: list of outgoing network interfaces + map_from: source network address + map_to: destination network address + + Returns: + name of rule config that was edited + """ + validate_netmap_rule(name, src, dest, map_from, map_to) + uci.set('netmap', id, 'name', name) + uci.set('netmap', id, 'src', src) + uci.set('netmap', id, 'dest', dest) + uci.set('netmap', id, 'device_in', device_in) + uci.set('netmap', id, 'device_out', device_out) + uci.set('netmap', id, 'map_from', map_from) + uci.set('netmap', id, 'map_to', map_to) + uci.save('netmap') + return id + +def delete_netmap_rule(uci, id: str) -> str: + """ + Delete netmap rule from netmap config. + + Args: + uci: EUci pointer + id: id of rule to delete + + Returns: + name of rule config that was deleted + """ + if not uci.get('netmap', id, default=None): + raise utils.ValidationError("id", "rule_does_not_exists", id) + uci.delete('netmap', id) + uci.save('netmap') + return id + +def list_netmap_devices(uci) -> list: + """ + Get all network devices from ip command + + Args: + uci: EUci pointer + + Returns: + a list of all network devices + """ + devices = [] + try: + output = subprocess.run(["ip", "-j", "link", "show"], capture_output=True, text=True) + data = json.loads(output.stdout) + for device in data: + if device.get('ifname') != 'lo' and not device.get('ifname').startswith('ifb-'): + devices.append({"device": device['ifname'], "interface": utils.get_interface_from_device(uci, device['ifname'])}) + except: + pass + return devices + +# Objects + +def update_redirect_rules(uci): + """ + Update redirect rules with ipset field set and ns_src set. + + Args: + uci: EUci pointer + changed_sections: list of changed objects, each object is in the form of `/` + """ + for section in utils.get_all_by_type(uci, 'firewall', 'redirect'): + # fetch ns_src and ns_dst for each redirect rule + ns_src = uci.get('firewall', section, 'ns_src', default=None) + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + if ns_dst: + # if ns_dst is set, fetch the ip address from the object and set it in the redirect rule + # in case that the redirect is then deleted, no action is needed + ipaddr = objects.get_object_ip(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + if ns_src: + # if ns_src is set, check if it is a domain set + database, id = ns_src.split('/') + obj_type = uci.get(database, id) + if database == "objects" and obj_type == "domain": + # if it is a domain set, set the ipset field in the redirect rule + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src_net") + try: + uci.delete('firewall', f"{section}_ipset") + except: + pass + else: + # create a full ipset configuration for the redirect rule + uci.set('firewall', section, 'ipset', f"{section}_ipset") + uci.set('firewall', f"{section}_ipset", "ipset") + uci.set('firewall', f"{section}_ipset", "name", f"{section}_ipset") + uci.set('firewall', f"{section}_ipset", "match", "src_net") + uci.set('firewall', f"{section}_ipset", "enabled", "1") + uci.set('firewall', f"{section}_ipset", 'ns_link', f"firewall/{section}") + uci.set('firewall', f"{section}_ipset", "entry", objects.get_object_ips(uci, ns_src)) + + uci.save('firewall') + +def update_firewall_rules(uci): + """ + Update firewall rules with ipset field set and ns_src set. + + Args: + uci: EUci pointer + """ + for section in utils.get_all_by_type(uci, 'firewall', 'rule'): + keep_ipset = False + ns_src = uci.get('firewall', section, 'ns_src', default=None) + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + name = uci.get('firewall', section, 'name', default=None) + if ns_src: + if objects.is_domain_set(uci, ns_src): + keep_ipset = True + id = ns_src.split('/')[1] + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src") + try: + uci.delete('firewall', section, 'src_ip') + except: + pass + else: + ipaddr = objects.get_object_ips(uci, ns_src) + if ipaddr: + uci.set('firewall', section, 'src_ip', ipaddr) + + if ns_dst: + if objects.is_domain_set(uci, ns_dst): + keep_ipset = True + id = ns_dst.split('/')[1] + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} dst") + try: + uci.delete('firewall', section, 'dest_ip') + except: + pass + else: + ipaddr = objects.get_object_ips(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + + # delete ipset field if no domains are set + if not keep_ipset: + try: + uci.delete('firewall', section, 'ipset') + except: + pass + uci.save('firewall') + +def list_object_suggestions(uci, expand = False): + """ + Get all objects from objects config + + Args: + uci: EUci pointer + expand: if True, expand object details + + Returns: + a list of all objects, each object is a dict with keys value, label, type + """ + return objects.list_all_objects(uci, expand) + +def get_default_logging_options(uci): + """ + Get default logging options + + Args: + uci: EUci pointer + + Returns: + a dict with keys rule_log_limit, zone_log_limit, redirect_log_limit + """ + + ret = {} + defaults = uci.get('firewall', 'ns_defaults', 'rule_log_limit', default=None) + if defaults is None: + # Set defaults, if needed + uci.set('firewall', 'ns_defaults', 'defaults') + uci.set('firewall', 'ns_defaults', 'rule_log_limit', '1/s') + uci.set('firewall', 'ns_defaults', 'zone_log_limit', '5/s') + uci.set('firewall', 'ns_defaults', 'redirect_log_limit', '1/s') + uci.save('firewall') + + for key in ['rule_log_limit', 'zone_log_limit', 'redirect_log_limit']: + ret[key] = uci.get('firewall', 'ns_defaults', key) + + return ret + +def apply_default_logging_options(uci): + """ + Walk all rules, zones and redirect rules and set logging options to default + + Args: + uci: EUci pointer + + Returns: + None + """ + log_opts = get_default_logging_options(uci) + for section in uci.get_all("firewall"): + if uci.get('firewall', section) == 'rule' and uci.get('firewall', section, 'log', default='0') == '1': + uci.set('firewall', section, 'log_limit', log_opts['rule_log_limit']) + elif uci.get('firewall', section) == 'zone' and uci.get('firewall', section, 'log', default='0') == '1': + uci.set('firewall', section, 'log_limit', log_opts['zone_log_limit']) + elif uci.get('firewall', section) == 'redirect' and uci.get('firewall', section, 'log', default='0') == '1': + uci.set('firewall', section, 'log_limit', log_opts['redirect_log_limit']) + uci.save('firewall') \ No newline at end of file diff --git a/packages/python3-nethsec/src/nethsec/inventory/__init__.py b/packages/python3-nethsec/src/nethsec/inventory/__init__.py new file mode 100644 index 000000000..079a680f2 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/inventory/__init__.py @@ -0,0 +1,950 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + + +from math import floor +from euci import EUci +from nethsec import utils, mwan, users, firewall, objects +import os +import re +import csv +import subprocess +import configparser +import json +import hashlib +from socket import inet_ntoa +from struct import pack + +## Utilities + +def _run(cmd): + try: + proc = subprocess.run(cmd, shell=True, check=True, capture_output=True, text=True) + return proc.stdout.rstrip().lstrip() + except: + return '' + +def _run_status(cmd): + try: + proc = subprocess.run(cmd, shell=True, check=True, capture_output=True, text=True) + return proc.returncode + except: + return 1 + +def _run_json(cmd): + try: + return json.loads(_run(cmd)) + except: + return {} + +def _get_role(uci: EUci, interface): + for zone in utils.get_all_by_type(uci, 'firewall', 'zone'): + name = uci.get('firewall', zone, 'name') + networks = uci.get('firewall', zone, 'network', list=True, default=[]) + if interface in networks: + if name == "lan": + return "green" + elif name == "wan": + return "red" + else: + return name + +def _get_ip(interface, uci: EUci): + info = _run_json(f"ifstatus {interface}") + try: + return anonymize(info['ipv4-address'][0]['address'], uci) + except: + return '' + +def _get_mask(interface): + info = _run_json(f"ifstatus {interface}") + try: + m = info['ipv4-address'][0]['mask'] + bits = 0xffffffff ^ (1 << 32 - int(m)) - 1 + mask = inet_ntoa(pack(">I", bits)) + return mask + except: + return '' + +def _get_gateway(interface, uci: EUci): + info = _run_json(f"ifstatus {interface}") + try: + for r in info['route']: + if r['target'] == '0.0.0.0': + return anonymize(r['nexthop'], uci) + except: + return '' + +def _get_cpu_field(field, cpu_info): + for f in cpu_info: + if f['field'].startswith(field): + return f['data'] + + return '' + +def anonymize(value, uci: EUci): + if fact_subscription_status(uci).get('status', 'no') != "no": + return value + h = hashlib.sha256(value.encode()).hexdigest() + return f"anon-{h[:16]}" + +def parse_version(version_str): + # Remove "NethSecurity " prefix if present + if version_str.startswith('NethSecurity '): + version_str = version_str[13:] # len('NethSecurity ') = 13 + # Take only the part before "-" + version_str = version_str.split('-')[0] + # Convert to tuple of integers for comparison + try: + return tuple(int(x) for x in version_str.split('.')) + except (ValueError, AttributeError): + return () + +def get_networks(uci: EUci): + networks = {} + devices = utils.get_all_by_type(uci, 'network', 'device') + for section in utils.get_all_by_type(uci, 'network', 'interface'): + if section == "lan6" or section == "wan6": # skip IPv6 for now + continue + interface = uci.get_all('network', section) + device = interface.get("device") + if not device or device == "lo" or device.startswith("ipsec") or device.startswith("tun"): + continue + network = {"type": "ethernet", "name": device, "props": { "role": _get_role(uci, section), "ipaddr": _get_ip(section, uci), "netmask": _get_mask(section), "gateway": _get_gateway(section, uci)}} + # get bridge ports, exclude vlans over bridges + is_vlan = bool(re.search(r'\.\d+$', interface['device'])) # check if the device name ends with . + if interface['device'].startswith('br') and not is_vlan: + network["type"] = "bridge" + for d in devices: + if uci.get('network', d, 'name') == device: + network['props']["bridge"] = uci.get('network', d, 'ports', default=[]) + networks[device] = network + return networks + +def get_version(): + version = "" + with open('/etc/os-release', 'r') as file: + for line in file: + if line.startswith("VERSION_ID="): + version = line.split('=')[1].replace('"','').rstrip() + break + return version + +def get_product(): + product = "" + try: + with open('/sys/devices/virtual/dmi/id/product_name', 'r') as f: + product = f.read().strip() + except: + pass + if not product: + try: + with open('/etc/board.json', 'r') as f: + binfo = json.load(f) + product = binfo['model']['name'] + except: + product = "" + return product + +def is_virtual(): + cpu_info = _run_json('lscpu -J')['lscpu'] + return _get_cpu_field("Hypervisor vendor", cpu_info) if _get_cpu_field("Hypervisor vendor", cpu_info) else 'physical' + +def get_cpu_info(): + cpu_info = _run_json('lscpu -J')['lscpu'] + return { + "model": _get_cpu_field("Model name", cpu_info), + "architecture": _get_cpu_field("Architecture", cpu_info) + } + +def get_pci_info(): + # map kernel driver to device id + drivers = {} + for line in _run("find /sys | grep '.*/drivers/.*/0000:.*$' | cut -d'/' -f6,7").split('\n'): + try: + (driver,bus) = line.split("/0000:") + drivers[bus] = driver + except: + continue + + # lspci -n: 00:1b.0 0403: 8086:293e (rev 03) + # fields: bus class vendor:device revision + pci = {} + if os.path.isdir('/proc/bus/pci'): + for line in _run("lspci -n").split("\n"): + revision = '' + fields = line.split(" ", maxsplit=4) + (vendor, device) = fields[2].split(":") + if len(fields) > 3: + revision = fields[4] + pci[fields[0]] = {"class_id": fields[1].rstrip(":"), "vendor_id": vendor, "device_id": device, "revision": revision.strip(')')} + + # lspci -mm: 00:00.0 "Host bridge" "Intel Corporation" "82G33/G31/P35/P31 Express DRAM Controller" -p00 "Red Hat, Inc." "QEMU Virtual Machine" + for fields in csv.reader(_run("lspci -mm").split("\n"), delimiter=' ', quotechar='"'): + pci[fields[0]]['class_name'] = fields[1].strip('"') + pci[fields[0]]['vendor_name'] = fields[2] + pci[fields[0]]['device_name'] = fields[3] + pci[fields[0]]['driver'] = drivers.get(fields[0], '') + return pci + +def get_memory(): + result = {} + try: + with open('/proc/meminfo', 'r') as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split(':') + if len(parts) == 2: + key = parts[0].strip() + value = parts[1].strip().split()[0] # Get the number, skip the unit (kB) + result[key] = int(value) * 1024 # Convert from kB to bytes + except: + pass + if result: + result["MemUsed"] = result.get('MemTotal', 0) - result.get('MemFree', 0) - result.get('Buffers', 0) - result.get('Cached', 0) + result["SwapUsed"] = result.get('SwapTotal', 0) - result.get('SwapFree', 0) + return result + +def get_mount_points(): + result = {} + exclude_mounts = {'/rom', '/overlay', '/dev'} + seen_mounts = set() + + try: + output = _run("df -P") + lines = output.strip().split('\n') + # Skip header line + for line in lines[1:]: + if not line.strip(): + continue + parts = line.split() + if len(parts) >= 6: + mount_point = parts[5] + # Skip excluded mount points + if mount_point in exclude_mounts: + continue + # Skip duplicates (keep first occurrence) + if mount_point in seen_mounts: + continue + try: + total = int(parts[1]) * 1024 # Convert from 1K blocks to bytes + used = int(parts[2]) * 1024 + available = int(parts[3]) * 1024 + result[mount_point] = { + 'total_bytes': total, + 'used_bytes': used, + 'available_bytes': available + } + seen_mounts.add(mount_point) + except (ValueError, IndexError): + continue + except: + pass + + return result + +## Facts +## These functions are used to send info about system to legacy my.nethesis.it +## The format should not be changed until legacy my.nethesis.it is alive + +def fact_hotspot(uci: EUci): + enabled = uci.get('dedalo', 'config', 'disabled', default='1') == '0' + server = uci.get('dedalo', 'config', 'api_url', default='') + interface = uci.get('dedalo', 'config', 'interface', default='') + return { 'enabled': enabled, 'server': server, 'interface': interface } + +def fact_netifyd(uci: EUci): + try: + config = configparser.ConfigParser() + config.read('/etc/netifyd.conf') + enable_sink = config.get('netifyd', 'enable_sink') + except: + enable_sink = 'no' + return { 'enabled': enable_sink == 'yes' } + +def fact_flashstart(uci: EUci): + enabled = uci.get('flashstart', 'global', 'enabled', default='0') == '1' + bypass = len(uci.get('flashstart', 'global', 'bypass', list=True, default=[])) + pro_plus = uci.get('flashstart', 'global', 'proplus', dtype=bool, default=False) + custom_servers = len(uci.get('flashstart', 'global', 'custom_servers', list=True, default=[])) + return { 'enabled': enabled, 'bypass': bypass, 'pro_plus': pro_plus, 'custom_servers': custom_servers } + +def fact_openvpn_rw(uci: EUci): + ret = { 'enabled': 0, 'server': 0, 'instances': [] } + for section in utils.get_all_by_type(uci, 'openvpn', 'openvpn'): + if uci.get("openvpn", section, 'ns_auth_mode', default=''): + ret["server"] += 1 + if uci.get("openvpn", section, 'enabled', default='0') == '1': + ret["enabled"] += 1 + for section in utils.get_all_by_type(uci, 'openvpn', 'openvpn'): + vpn = uci.get_all("openvpn", section) + if not section.startswith('ns_'): + continue + if "ns_auth_mode" in vpn: + # we are in a ovpn_rw + instance = { + 'section': section, + 'authentication': vpn.get('ns_auth_mode'), + 'user_database': vpn.get('ns_user_db'), + 'mode': vpn.get('dev_type') + } + ret['instances'].append(instance) + return ret + +def fact_openvpn_tun(uci: EUci): + ret = { 'client': 0, 'server': 0, 'tunnels': [] } + for section in utils.get_all_by_type(uci, 'openvpn', 'openvpn'): + vpn = uci.get_all("openvpn", section) + if 'ns_auth_mode' in vpn or not section.startswith('ns_'): + continue + if vpn.get("client", "0") == "1" or vpn.get("ns_client", "0") == "1": + ret["client"] += 1 + else: + ret["server"] += 1 + instance = { + 'section': section, + 'mode': vpn.get('dev_type') + } + ret['tunnels'].append(instance) + return ret + +def fact_certificates_info(uci: EUci): + result = { + "custom_certificates": { + "count": 0 + }, + "acme_certificates": { + "count": 0, + "issued": 0, + "pending": 0 + } + } + + # Count custom certificates + try: + for entry in os.scandir('/etc/nginx/custom_certs'): + if entry.is_file() and entry.name.endswith('.crt') and os.path.isfile(entry.path[:-4] + '.key'): + result["custom_certificates"]["count"] += 1 + except Exception as e: + # Handle exceptions appropriately + pass + + # Count ACME certificates + try: + requested_certificates = utils.get_all_by_type(uci, 'acme', 'cert') + enabled_certificates = [certificate for certificate in requested_certificates + if requested_certificates[certificate]['enabled'] == '1'] + for certificate in enabled_certificates: + result["acme_certificates"]["count"] += 1 + domain = requested_certificates[certificate]['domains'][0] + cert_path = f'/etc/ssl/acme/{domain}.fullchain.crt' + if os.path.isfile(cert_path): + result["acme_certificates"]["issued"] += 1 + else: + result["acme_certificates"]["pending"] += 1 + except Exception as e: + # Handle exceptions appropriately + pass + + return result + +def fact_subscription_status(uci: EUci): + return { 'status': uci.get('ns-plug', 'config', 'type', default='no') } + +def fact_controller(uci: EUci): + if uci.get('ns-plug', 'config', 'server', default='') and uci.get('ns-plug', 'config', 'unit_id', default='') and uci.get('ns-plug', 'config', 'token', default=''): + return { "enabled": True} + else: + return { "enabled": False} + +def fact_rpcd_users(uci: EUci): + count = 0 + for section, options in (utils.get_all_by_type(uci, 'rpcd', 'login') or {}).items(): + # skip root and the controller user (controller has a random username) + if options.get('username') == 'root' or section == 'controller': + continue + count += 1 + return { 'count': count } + +def fact_threat_shield(uci: EUci): + ret = { 'enabled': False, 'community': 0, 'enterprise': 0 } + ret['enabled'] = uci.get('banip', 'global', 'ban_enabled', default='0') == '1' + try: + for feed in uci.get_all("banip", "global", "ban_feed"): + if feed.startswith("nethesis") or feed.startswith("yoroi"): + ret['enterprise'] += 1 + else: + ret['community'] += 1 + except: + pass + return ret + +def fact_adblock(uci: EUci): + ret = { 'enabled': False, 'community': 0, 'enterprise': 0 } + ret['enabled'] = uci.get('adblock', 'global', 'ts_enabled', default='0') == '1' + try: + enabled_feeds = list(uci.get_all('adblock', 'global', 'adb_sources')) + except: + enabled_feeds = [] + for feed in enabled_feeds: + if feed.startswith("nethesis") or feed.startswith("yoroi"): + ret['enterprise'] += 1 + else: + ret['community'] += 1 + return ret + +def fact_ui(uci: EUci): + ret = { 'luci': False, 'port443': False, 'port9090': False } + ret['luci'] = uci.get('ns-ui', 'config', 'luci_enable', default='0') == '1' + ret['port443'] = uci.get('ns-ui', 'config', 'nsui_enable', default='0') == '1' + ret['port9090'] = uci.get('ns-ui', 'config', 'nsui_extra_enable', default='0') == '1' and uci.get('ns-ui', 'config', 'nsui_extra_port', default='0') == '9090' + return ret + +def fact_network(uci: EUci): + result: dict = { + "zones": [] + } + vlan_count = 0 + bridge_count = 0 + bond_count = 0 + zone_network_counts = {} + route_info = { + "count_ipv6_route": 0, + "count_ipv4_route": 0 + } + # Regex pattern to match interfaces that end with "." + vlan_pattern = re.compile(r'\.\d+$') + interfaces = utils.get_all_by_type(uci, 'network', 'interface') + + # Loop through all firewall zones to gather network information + for zone in utils.get_all_by_type(uci, 'firewall', 'zone').values(): + zone_info = { + 'name': zone['name'], + 'ipv4': 0, + 'ipv6': 0 + } + devices = utils.get_all_devices_by_zone(uci, zone['name'], True) + for device in devices: + interface = utils.get_interface_from_device(uci, device) + if interface is None: + continue + is_ipv6 = False + for option in uci.get_all('network', interface): + if option.startswith("ip6") or option == "dhcpv6" or option == "ipv6": + is_ipv6 = True + break + if uci.get('network', interface, 'proto', default="") in ['dhcpv6', '6in4', '6to4', '6rd', 'grev6', 'grev6tap', 'vtiv6']: + is_ipv6 = True + if is_ipv6: + zone_info['ipv6'] += 1 + else: + zone_info['ipv4'] += 1 + # Count VLAN, bridge, and bond interfaces + if 'device' in interfaces.get(interface, {}): + device_name = interfaces[interface]['device'] + if vlan_pattern.search(device_name): + vlan_count += 1 + if device_name.startswith('br-'): + bridge_count += 1 + if device_name.startswith('bond-'): + bond_count += 1 + result["zones"].append(zone_info) + # Count networks for each zone + networks = uci.get('firewall', 'ns_'+zone['name'], 'network', list=True, default=[]) + network_count = len(networks) + # Count devices for each zone (if networks are not defined, hotspot zones, openvpn zones, etc.) + devices = utils.get_all_devices_by_zone(uci, zone['name'], True) + # remove tun-dedalo if present (it's a virtual device, we count the real interface + devices = [d for d in devices if not d.startswith('tun-dedalo')] + devices_count = len(devices) + zone_network_counts[zone['name']] = network_count or devices_count # Use the number of networks if available, otherwise use the number of devices + # Get route information + routes_ipv6 = utils.get_all_by_type(uci, 'network', 'route6') + for _ in routes_ipv6: + route_info["count_ipv6_route"] += 1 + + routes_ipv4 = utils.get_all_by_type(uci, 'network', 'route') + for _ in routes_ipv4: + route_info["count_ipv4_route"] += 1 + + # Add VLAN, bridge, and bond counts to the result + result['interface_counts'] = { + 'vlans': vlan_count, + 'bridges': bridge_count, + 'bonds': bond_count + } + # Add network zone counts to the result + result['zone_network_counts'] = zone_network_counts + + # Add route information to the result + result['route_info'] = route_info + + result['configuration'] = get_networks(uci) + + return result + +def fact_database_stats(uci: EUci): + ret = {} + databases = users.list_databases(uci) + for db in databases: + name = db["name"] + number_users = len(users.list_users(uci, name)) + ret[name] = { "users": number_users } + return ret + +def fact_firewall_stats(uci: EUci): + result = { + "firewall": { + "port_forward": 0, + "nat": {"masquerade": 0, "snat": 0, "accept": 0}, + "netmap": {"source": 0, "destination": 0}, + "rules": {"forward": 0, "input": 0, "output": 0} + }, + "objects": { + "domains": 0, + "hosts": 0, + "port_forward": {"allowed_from": 0, "destination_to": 0}, + "mwan_rules": 0, + "rules": {"forward": 0, "input": 0, "output": 0} + } + } + + # Firewall Information + # Count port forward + result["firewall"]["port_forward"] = len(utils.get_all_by_type(uci, 'firewall', 'redirect')) + + # Count NAT rules + for rule in firewall.list_nat_rules(uci): + if rule['target'] == 'ACCEPT': + result["firewall"]["nat"]["accept"] += 1 + elif rule['target'] == 'MASQUERADE': + result["firewall"]["nat"]["masquerade"] += 1 + elif rule['target'] == 'SNAT': + result["firewall"]["nat"]["snat"] += 1 + + # Count netmap rules + for rule in firewall.list_netmap_rules(uci): + if rule.get('dest', ''): + result["firewall"]["netmap"]["source"] += 1 + elif rule.get('src', ''): + result["firewall"]["netmap"]["destination"] += 1 + + # Count rules + result["firewall"]["rules"]["forward"] = len(firewall.list_forward_rules(uci)) + result["firewall"]["rules"]["input"] = len(firewall.list_input_rules(uci)) + result["firewall"]["rules"]["output"] = len(firewall.list_output_rules(uci)) + + # Object Information + # Count objects + result["objects"]["domains"] = len(objects.list_domain_sets(uci)) + result["objects"]["hosts"] = len(objects.list_host_sets(uci)) + + # Count object for port forward + for key, value in utils.get_all_by_type(uci, 'firewall', 'redirect').items(): + if isinstance(value.get('ns_src'), str): + result["objects"]["port_forward"]["allowed_from"] += 1 + if isinstance(value.get('ns_dst'), str): + result["objects"]["port_forward"]["destination_to"] += 1 + + # Count object for rules + for value in firewall.list_forward_rules(uci): + if isinstance(value.get('ns_dst'), str): + result["objects"]["rules"]["forward"] += 1 + for value in firewall.list_input_rules(uci): + if isinstance(value.get('ns_src'), str): + result["objects"]["rules"]["input"] += 1 + for value in firewall.list_output_rules(uci): + if isinstance(value.get('ns_dst'), str): + result["objects"]["rules"]["output"] += 1 + for value in mwan.index_rules(uci): + if isinstance(value.get('ns_src'), str): + result["objects"]["mwan_rules"] += 1 + + return result + +def fact_storage(uci: EUci): + return {"enabled": uci.get("fstab", "ns_data", "enabled", default="0") == "1"} + +def fact_clm(uci: EUci): + return {"enabled": uci.get("ns-clm", "config", "enabled", default="0") == "1"} + +def fact_proxy_pass(uci: EUci): + ret = { "count": 0} + try: + for l in utils.get_all_by_type(uci, 'nginx', 'location'): + if uci.get('nginx', l, 'proxy_pass', default=''): + ret["count"] += 1 + except: + pass + return ret + +def fact_dpi(uci: EUci): + ret = {"enabled": False, "rules": 0} + ret["enabled"] = uci.get('dpi', 'config', 'enabled', default='0') == '1' + for rule in utils.get_all_by_type(uci, 'dpi', 'rule'): + if uci.get('dpi', rule, 'enabled', default='0') == '1': + ret["rules"] += 1 + return ret + +def fact_dhcp_server(uci: EUci): + result = { + 'count': 0, # Initialize the count for DHCP servers + 'static_leases': 0, + 'dynamic_leases': 0, + 'dns_records_count': 0, + 'dns_forwarder_enabled': False + } + + # Count DHCP servers + for section in utils.get_all_by_type(uci, 'dhcp', 'dhcp'): + if uci.get('dhcp', section, 'dhcpv4', default='') == 'server' or uci.get('dhcp', section, 'dhcpv6', default='') == 'server': + result['count'] += 1 # Increment the count for DHCP servers + + # Count static leases + result['static_leases'] = len(utils.get_all_by_type(uci, 'dhcp', 'host')) + + static_leases = [] + for l in utils.get_all_by_type(uci, 'dhcp', 'host'): + ldata = uci.get_all('dhcp', l) + if 'mac' in ldata and 'ip' in ldata: + static_leases.append(ldata['mac'].lower()) + + # Count dynamic leases, skipping static leases + try: + with open("/tmp/dhcp.leases", "r") as fp: + for line in fp.readlines(): + tmp = line.split(" ") + if tmp[1].lower() not in static_leases: + result['dynamic_leases'] += 1 + except FileNotFoundError: + # Handle the case where the leases file doesn't exist + pass + + # Count DNS records and check if DNS forwarder is enabled + for section in utils.get_all_by_type(uci, 'dhcp', 'dnsmasq'): + servers = uci.get('dhcp', section, 'server', default=[]) + if servers: + result['dns_forwarder_enabled'] = True + for r in utils.get_all_by_type(uci, 'dhcp', 'domain'): + result['dns_records_count'] += 1 + + return result + +def fact_multiwan(uci: EUci): + policies = mwan.index_policies(uci) + result = { + 'enabled': len(policies) > 0, + 'policies': { + 'backup': 0, + 'balance': 0, + 'custom': 0 + }, + 'rules': len(mwan.index_rules(uci)) + } + for policy in policies: + result['policies'][policy['type']] += 1 + return result + +def fact_qos(uci: EUci): + ret = { + "count": 0, + "rules": [] + } + for key, interface in utils.get_all_by_type(uci, 'qosify', 'interface').items(): + if interface['disabled'] == '0': + ret["count"] += 1 + rule = { + 'enabled': interface['disabled'] == '0', + 'upload': int(interface['bandwidth_up'].removesuffix('mbit')), + 'download': int(interface['bandwidth_down'].removesuffix('mbit')), + } + ret['rules'].append(rule) + return ret + +def fact_ipsec(uci: EUci): + try: + count = len(utils.get_all_by_type(uci, 'ipsec', 'remote')) + except: + count = -1 + return { 'count': count } + +def fact_nathelpers(uci: EUci): + # count the number of lines in the file + try: + with open('/etc/modules.d/ns-nathelpers') as f: + count = len(f.readlines()) + except: + count = 0 + return { 'count': count, 'enabled': count > 0 } + +def fact_ddns(uci: EUci): + ddns = _run_status("/etc/init.d/ddns enabled") + return { 'enabled': ddns == 0 } + +def fact_snmp (uci: EUci): + snmp = _run_status("/etc/init.d/snmpd running") + return { 'enabled': snmp == 0 } + +def fact_wireguard(uci: EUci): + servers = dict() + interfaces = utils.get_all_by_type(uci, "network", "interface") + for i in interfaces: + if interfaces[i].get("proto", "") != "wireguard": + continue + peers = utils.get_all_by_type(uci, 'network', f'wireguard_{i}') + routing_all_traffic = 0 + for peer in peers: + if uci.get('network', peer, 'ns_route_all_traffic', dtype=bool, default=False): + routing_all_traffic += 1 + servers[i] = { + "peers": len(peers), + "routing_all_traffic": routing_all_traffic + } + + return { + "enabled": len(servers) > 0, + "servers": servers + } + + +def fact_snort(uci: EUci): + ret = { 'enabled': False, 'policy': '', 'oink_enabled': False, 'disabled_rules': 0, 'suppressed_rules': 0, 'bypass_src_ipv4': 0, 'bypass_src_ipv6': 0, 'bypass_dst_ipv4': 0, 'bypass_dst_ipv6': 0 } + + ret['enabled'] = uci.get('snort', 'snort', 'enabled', dtype=bool, default=False) + ret['policy'] = uci.get('snort', 'snort', 'ns_policy', default='') + ret['oink_enabled'] = True if uci.get('snort', 'snort', 'oinkcode', default='') else False + + # count list of ns_disabled_rules + ret['disabled_rules'] = len(uci.get('snort', 'snort', 'ns_disabled_rules', list=True, default=[])) + # count list of ns_suppress rules + ret['suppressed_rules'] = len(uci.get('snort', 'snort', 'ns_suppress', list=True, default=[])) + # count the source bypass of ipv4 and ipv6 + ret['bypass_src_ipv4'] = len(uci.get('snort', 'nfq', 'bypass_src_v4', list=True, default=[])) + ret['bypass_src_ipv6'] = len(uci.get('snort', 'nfq', 'bypass_src_v6', list=True, default=[])) + ## count the destination bypass_dst_v4 and bypass_dst_ipv6 + ret['bypass_dst_ipv4'] = len(uci.get('snort', 'nfq', 'bypass_dst_v4', list=True, default=[])) + ret['bypass_dst_ipv6'] = len(uci.get('snort', 'nfq', 'bypass_dst_v6', list=True, default=[])) + + return ret + +def fact_mac_ip_binding(uci: EUci): + ret = { "disabled": 0, "soft-binding": 0, "hard-binding": 0 } + # Parse DHCP servers (interface with static IP, wan is excluded) + result = subprocess.check_output(['/usr/libexec/rpcd/ns.dhcp', 'call', 'list-interfaces']) + interfaces = json.loads(result) + for i in interfaces: + if uci.get('dhcp', i, 'ns_binding', default='') == '' or uci.get('dhcp', i, 'ns_binding', default='') == '0': + ret['disabled'] += 1 # Increment the count for DHCP servers + elif uci.get('dhcp', i, 'ns_binding', default='') == '1': + ret['soft-binding'] += 1 + elif uci.get('dhcp', i, 'ns_binding', default='') == '2': + ret['hard-binding'] += 1 + return ret + +def fact_backups(uci: EUci): + ret = { 'backup_passphrase': False, 'passphrase_date': 0 } + try: + ret['backup_passphrase'] = os.path.exists('/etc/backup.pass') + ret['passphrase_date'] = int(os.path.getmtime('/etc/backup.pass')) + except: + return {} + return ret + +def fact_ha(uci: EUci): + vrrp_instances = utils.get_all_by_type(uci, 'keepalived', 'vrrp_instance') + if vrrp_instances is None: + vrrp_instances = [] + ipaddresses = utils.get_all_by_type(uci, 'keepalived', 'ipaddress') + if ipaddresses is None: + ipaddresses = [] + + return { 'enabled': len(vrrp_instances) > 0, 'vips': len(ipaddresses) } + +def fact_default_password(uci: EUci): + data = { + 'username': 'root', + 'password': 'Nethesis,1234', + 'timeout': 1 + } + result = subprocess.run(['/bin/ubus', 'call', 'session', 'login', json.dumps(data)], capture_output=True) + + return { 'default_password': result.returncode == 0 } + +## Info +## These functions are used to send info about system to new my.nethesis.it and phonehome.nethserver.org + +def info_fqdn(uci: EUci): + system = uci.get_all('system') + for section in system: + for option in system[section]: + if option == 'hostname': + return anonymize(system[section][option], uci) + return '' + +def info_timezone(uci: EUci): + system = uci.get_all('system') + for section in system: + for option in system[section]: + if option == 'zonename': + return system[section][option] + return '' + +def info_kernel_version(uci: EUci): + try: + with open('/proc/version', 'r') as f: + version = f.read().strip() + return version.split()[2] + except: + return '' + +def info_uptime_seconds(uci: EUci): + try: + with open('/proc/uptime', 'r') as f: + uptime = f.read().strip().split()[0] + return floor(float(uptime)) + except: + return 0 + +def info_default_ipv4(uci: EUci): + # first method: dig -4 TXT +short o-o.myaddr.l.google.com @ns1.google.com + try: + res = subprocess.run(['dig', '-4', '+short', 'myip.opendns.com', '@resolver1.opendns.com'], + capture_output=True, text=True, timeout=3) + if res.returncode == 0 and res.stdout.strip(): + ip = res.stdout.strip().strip('"') + if ip: + return anonymize(ip, uci) + except: + pass + + # second method: curl -4 ifconfig.co + try: + res = subprocess.run(['curl', '-4', '-s', 'ifconfig.co'], + capture_output=True, text=True, timeout=3) + if res.returncode == 0 and res.stdout.strip(): + return anonymize(res.stdout.strip(), uci) + except: + pass + + # third method: get the first WAN device and its IPv4 + try: + wan_devices = utils.get_all_wan_devices(uci) + if wan_devices: + first_wan = wan_devices[0] + res = subprocess.run(['ip', '-4', '-j', 'addr', 'show', first_wan], + capture_output=True, text=True, timeout=3) + if res.returncode == 0: + addr_info = json.loads(res.stdout) + if addr_info and 'addr_info' in addr_info[0]: + for addr in addr_info[0]['addr_info']: + if addr.get('family') == 'inet': + return anonymize(addr.get('local'), uci) + except: + pass + + return '' + +def info_default_ipv6(uci: EUci): + # Get the first WAN device and its IPv6 + ipv6 = '' + try: + wan_devices = utils.get_all_wan_devices(uci) + if wan_devices: + first_wan = wan_devices[0] + res = subprocess.run(['ip', '-6', '-j', 'addr', 'show', first_wan], + capture_output=True, text=True, timeout=3) + if res.returncode == 0: + addr_info = json.loads(res.stdout) + if addr_info and 'addr_info' in addr_info[0]: + for addr in addr_info[0]['addr_info']: + if addr.get('family') == 'inet6' and addr.get('scope') == 'global': + ipv6 = addr.get('local') + except: + pass + + # If WAN has no IPv6, assume that IPv6 is not configured: speedup data collection + if ipv6 == '': + return '' + + # first method: dig -6 TXT +short o-o.myaddr.l.google.com @ns1.google.com + try: + res = subprocess.run(['dig', '-6', 'TXT', '+short', 'o-o.myaddr.l.google.com', '@ns1.google.com'], + capture_output=True, text=True, timeout=3) + if res.returncode == 0 and res.stdout.strip(): + ip = res.stdout.strip().strip('"') + if ip: + return anonymize(ip, uci) + except: + pass + + # second method: curl -6 ifconfig.co + try: + res = subprocess.run(['curl', '-6', '-s', 'ifconfig.co'], + capture_output=True, text=True, timeout=3) + if res.returncode == 0 and res.stdout.strip(): + return anonymize(res.stdout.strip(), uci) + except: + pass + + + return anonymize(ipv6, uci) + +def info_package_updates_available(uci: EUci): + """Check if package updates are available""" + try: + res = subprocess.run(['/usr/libexec/rpcd/ns.update', 'call', 'check-package-updates'], + capture_output=True, text=True, timeout=10) + if res.returncode == 0: + data = json.loads(res.stdout) + if isinstance(data, dict) and 'updates' in data: + updates = data['updates'] + if isinstance(updates, list) and len(updates) > 0: + return True + except: + pass + return False + +def info_image_updates_available(uci: EUci): + """Check if system image updates are available""" + try: + res = subprocess.run(['/usr/libexec/rpcd/ns.update', 'call', 'check-system-update'], + capture_output=True, text=True, timeout=10) + if res.returncode == 0: + data = json.loads(res.stdout) + current_version = data.get('currentVersion', '') + last_version = data.get('lastVersion', '') + + if current_version and last_version: + current = parse_version(current_version) + last = parse_version(last_version) + if current and last and last > current: + return True + except: + pass + return False + +def info_dns_servers(uci: EUci): + dns_list = [] + try: + for server in uci.get('dhcp', 'ns_dnsmasq', 'server', list=True, default=[]): + dns_list.append(server) + except: + pass + if not dns_list: + try: + with open('/tmp/resolv.conf.d/resolv.conf.auto', 'r') as f: + for line in f: + line = line.strip() + if line.startswith('nameserver'): + parts = line.split() + if len(parts) > 1: + dns_list.append(parts[1]) + except: + pass + return dns_list diff --git a/packages/python3-nethsec/src/nethsec/ipsec/__init__.py b/packages/python3-nethsec/src/nethsec/ipsec/__init__.py new file mode 100644 index 000000000..254966e6c --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/ipsec/__init__.py @@ -0,0 +1,93 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +IPSec utilities +''' + +import os +from nethsec import utils, firewall + +IPSEC_ZONE='ipsec' + +def init_ipsec(uci): + ''' + Initialize IPSec global configuration, if needed. + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + ''' + # Make sure the config file exists + conf = os.path.join(uci.confdir(), 'ipsec') + if not os.path.isfile(conf): + with open(conf, 'a'): + pass + + # Setup global options + gsettings = utils.get_id("ipsec_global") + uci.set("ipsec", gsettings, IPSEC_ZONE) + uci.set("ipsec", gsettings, "debug", '0') + uci.set("ipsec", gsettings, "zone", 'ipsec') + uci.commit('ipsec') + +def open_firewall_ports(uci): + ''' + Open firewall ports for IPSec tunnels, if need. + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + ''' + esp_accepted = False + ike_accepted = False + nat_accepted = False + esp = {"src": "wan", "dest_port": "", "proto": "esp", "target": "ACCEPT"} + ike = {"src": "wan", "dest_port": "500", "proto": "udp", "target": "ACCEPT"} + nat = {"src": "wan", "dest_port": "4500", "proto": "udp", "target": "ACCEPT"} + # search for existing rules + for r in utils.get_all_by_type(uci, 'firewall', 'rule'): + tmp = dict() + for opt in ['src', 'dest', 'dest_port', 'proto', 'target']: + tmp[opt] = uci.get('firewall', r, opt, default='') + # check if tmp is the esp rule + if all((tmp.get(k) == v for k, v in esp.items())): + esp_accepted = True + # check if tmp is the ike rule + if all((tmp.get(k) == v for k, v in ike.items())): + ike_accepted = True + # check if tmp is the nat rule + if all((tmp.get(k) == v for k, v in nat.items())): + nat_accepted = True + + if not ike_accepted: + firewall.add_template_rule(uci, 'ns_ipsec_ike') + + if not esp_accepted: + firewall.add_template_rule(uci, 'ns_ipsec_esp') + + if not nat_accepted: + firewall.add_template_rule(uci, 'ns_ipsec_nat') + + if not nat_accepted or not ike_accepted or not esp_accepted: + uci.save('firewall') + +def add_trusted_interface(uci, interface): + ''' + Add the interface to the 'ipsec' trusted zone. The function also creates the trusted zone, if needed. + + Changes are saved to staging area. + + Arguments: + - uci -- EUci pointer + ''' + if firewall.zone_exists(uci, IPSEC_ZONE): + firewall.add_interface_to_zone(uci, interface, IPSEC_ZONE) + else: + firewall.add_trusted_zone(uci, IPSEC_ZONE, [interface]) diff --git a/packages/python3-nethsec/src/nethsec/ldif/__init__.py b/packages/python3-nethsec/src/nethsec/ldif/__init__.py new file mode 100644 index 000000000..0c5e4c6b1 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/ldif/__init__.py @@ -0,0 +1,418 @@ +# Imported from: https://github.com/abilian/ldif + +# Copyright (c) 2019-2021, Abilian SAS +# Copyright (c) 2015, Tobias Bengfort +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""ldif - generate and parse LDIF data (see RFC 2849).""" + + +import base64 +import logging +import re +from collections import OrderedDict +from collections.abc import Collection, Iterator +from typing import Optional, Union +from urllib.parse import urlparse +from urllib.request import urlopen + +__version__ = "4.1.0" + +__all__ = [ + # constants + "LDIF_PATTERN", + # classes + "LDIFWriter", + "LDIFParser", +] + +log = logging.getLogger("ldif") + +ATTRTYPE_PATTERN = r"[\w;.-]+(;[\w_-]+)*" +ATTRVALUE_PATTERN = r'(([^,]|\\,)+|".*?")' +ATTR_PATTERN = ATTRTYPE_PATTERN + r"[ ]*=[ ]*" + ATTRVALUE_PATTERN +RDN_PATTERN = ATTR_PATTERN + r"([ ]*\+[ ]*" + ATTR_PATTERN + r")*[ ]*" +DN_PATTERN = RDN_PATTERN + r"([ ]*,[ ]*" + RDN_PATTERN + r")*[ ]*" +DN_REGEX = re.compile(f"^{DN_PATTERN}$") + +LDIF_PATTERN = ( + f"^((dn(:|::) {vars()['DN_PATTERN']})|(%(ATTRTYPE_PATTERN) s(:|::) .*)$)+" +) + +MOD_OPS = ["add", "delete", "replace"] +CHANGE_TYPES = ["add", "delete", "modify", "modrdn"] + + +def is_dn(s: str) -> bool: + """Return True if s is a LDAP DN.""" + if s == "": + return True + rm = DN_REGEX.match(s) + return rm is not None and rm.group(0) == s + + +UNSAFE_STRING_PATTERN = ( + "(^[^\x01-\x09\x0b-\x0c\x0e-\x1f\x21-\x39\x3b\x3d-\x7f]" + "|[^\x01-\x09\x0b-\x0c\x0e-\x7f])" +) +UNSAFE_STRING_RE = re.compile(UNSAFE_STRING_PATTERN) + + +def lower(string_list: Optional[Collection[str]]) -> list[str]: + """Return a list with the lowercased items of l.""" + return [i.lower() for i in string_list or []] + + +class LDIFWriter: + """Write LDIF entry or change records to file object. + + :type output_file: file-like object in binary mode + :param output_file: File for output + + :type base64_attrs: List[string] + :param base64_attrs: List of attribute types to be base64-encoded in any + case + + :type cols: int + :param cols: Specifies how many columns a line may have before it is + folded into many lines + + :type line_sep: bytearray + :param line_sep: line separator + + :type encoding: string + :param encoding: Encoding to use for converting values to bytes. Note that + the spec requires the dn field to be UTF-8 encoded, so it does not + really make sense to use anything else. Default: ``'utf8'``. + """ + + def __init__( + self, output_file, base64_attrs=(), cols=76, line_sep=b"\n", encoding="utf8" + ): + self._output_file = output_file + self._base64_attrs = lower(base64_attrs) + self._cols = cols + self._line_sep = line_sep + self._encoding = encoding + + self.records_written = 0 #: number of records that have been written + + def _fold_line(self, line: bytes) -> None: + """Output string line as one or more folded lines.""" + if len(line) <= self._cols: + self._output_file.write(line) + self._output_file.write(self._line_sep) + else: + pos = self._cols + self._output_file.write(line[0 : self._cols]) + self._output_file.write(self._line_sep) + while pos < len(line): + self._output_file.write(b" ") + end = min(len(line), pos + self._cols - 1) + self._output_file.write(line[pos:end]) + self._output_file.write(self._line_sep) + pos = end + + def _needs_base64_encoding(self, attr_type, attr_value): + """Return True if attr_value has to be base-64 encoded. + + This is the case because of special chars or because attr_type + is in self._base64_attrs + """ + return ( + attr_type.lower() in self._base64_attrs + or isinstance(attr_value, bytes) + or UNSAFE_STRING_RE.search(attr_value) is not None + ) + + def _unparse_attr(self, attr_type, attr_value): + """Write a single attribute type/value pair.""" + if self._needs_base64_encoding(attr_type, attr_value): + if not isinstance(attr_value, bytes): + attr_value = attr_value.encode(self._encoding) + encoded = base64.encodebytes(attr_value).replace(b"\n", b"").decode("ascii") + line = ":: ".join([attr_type, encoded]) + else: + line = ": ".join([attr_type, attr_value]) + self._fold_line(line.encode("ascii")) + + def _unparse_entry_record(self, entry): + """ + :type entry: Dict[string, List[string]] + :param entry: Dictionary holding an entry + """ + for attr_type in entry.keys(): + for attr_value in entry[attr_type]: + self._unparse_attr(attr_type, attr_value) + + def _unparse_changetype(self, mod_len): + """Detect and write the changetype.""" + if mod_len == 2: + changetype = "add" + elif mod_len == 3: + changetype = "modify" + else: + raise ValueError("modlist item of wrong length") + + self._unparse_attr("changetype", changetype) + + def _unparse_change_record(self, modlist): + """ + :type modlist: List[Tuple] + :param modlist: List of additions (2-tuple) or modifications (3-tuple) + """ + mod_len = len(modlist[0]) + self._unparse_changetype(mod_len) + + for mod in modlist: + if len(mod) != mod_len: + raise ValueError("Subsequent modlist item of wrong length") + + if mod_len == 2: + mod_type, mod_vals = mod + elif mod_len == 3: + mod_op, mod_type, mod_vals = mod + self._unparse_attr(MOD_OPS[mod_op], mod_type) + else: + raise ValueError(f"modlist item of wrong length: {mod_len}") + + for mod_val in mod_vals: + self._unparse_attr(mod_type, mod_val) + + if mod_len == 3: + self._output_file.write(b"-" + self._line_sep) + + def unparse(self, dn: str, record: Union[list, dict]): + """Write an entry or change record to the output file. + + :type dn: string + :param dn: distinguished name + :type record: Union[Dict[string, List[string]], List[Tuple]] + :param record: Either a dictionary holding an entry or a list of + additions (2-tuple) or modifications (3-tuple). + """ + self._unparse_attr("dn", dn) + if isinstance(record, dict): + self._unparse_entry_record(record) + elif isinstance(record, list): + self._unparse_change_record(record) + else: + raise TypeError("Argument record must be dictionary or list") + self._output_file.write(self._line_sep) + self.records_written += 1 + + +class LDIFParser: + """Read LDIF entry or change records from file object. + + :type input_file: file-like object in binary mode + :param input_file: file to read the LDIF input from + + :type ignored_attr_types: List[string] + :param ignored_attr_types: List of attribute types that will be ignored + + :type process_url_schemes: List[bytearray] + :param process_url_schemes: List of URL schemes to process with urllib. + An empty list turns off all URL processing and the attribute is + ignored completely. + + :type line_sep: bytearray + :param line_sep: line separator + + :type encoding: string + :param encoding: Encoding to use for converting values to unicode strings. + If decoding failes, the raw bytestring will be used instead. You can + also pass ``None`` which will skip decoding and always produce + bytestrings. Note that this only applies to entry values. ``dn`` and + entry keys will always be unicode strings. + + :type strict: boolean + :param strict: If set to ``False``, recoverable parse errors will produce + log warnings rather than exceptions. + """ + + def __init__( + self, + input_file, + ignored_attr_types=(), + process_url_schemes=(), + line_sep=b"\n", + encoding: Optional[str] = "utf8", + strict=True, + ): + self._input_file = input_file + self._process_url_schemes = lower(process_url_schemes) + self._ignored_attr_types = lower(ignored_attr_types) + self._line_sep = line_sep + self._encoding = encoding + self._strict = strict + + self.line_counter = 0 #: number of lines that have been read + self.byte_counter = 0 #: number of bytes that have been read + self.records_read = 0 #: number of records that have been read + + def parse(self) -> Iterator[tuple[Optional[str], dict]]: + """Iterate LDIF entry records. + + :rtype: Iterator[Tuple[string, Dict]] + :return: (dn, entry) + """ + for block in self._iter_blocks(): + yield self._parse_entry_record(block) + + @staticmethod + def _strip_line_sep(s): + """Strip trailing line separators from s, but no other whitespaces.""" + if s[-2:] == b"\r\n": + return s[:-2] + elif s[-1:] == b"\n": + return s[:-1] + else: + return s + + def _iter_unfolded_lines(self): + """Iter input unfoled lines. + + Skip comments. + """ + line = self._input_file.readline() + while line: + self.line_counter += 1 + self.byte_counter += len(line) + + line = self._strip_line_sep(line) + + nextline = self._input_file.readline() + while nextline and nextline[:1] == b" ": + line += self._strip_line_sep(nextline)[1:] + nextline = self._input_file.readline() + + if not line.startswith(b"#"): + yield line + line = nextline + + def _iter_blocks(self): + """Iter input lines in blocks separated by blank lines.""" + lines = [] + for line in self._iter_unfolded_lines(): + if line: + lines.append(line) + elif lines: + self.records_read += 1 + yield lines + lines = [] + if lines: + self.records_read += 1 + yield lines + + def _decode_value(self, attr_type, attr_value): + if attr_type == "dn": + try: + return attr_type, attr_value.decode("utf8") + except UnicodeError as err: + self._error(err) + return attr_type, attr_value.decode("utf8", "ignore") + + elif self._encoding is not None: + try: + return attr_type, attr_value.decode(self._encoding) + except UnicodeError: + pass + + return attr_type, attr_value + + def _parse_attr(self, line: bytes): + """Parse a single attribute type/value pair.""" + colon_pos = line.index(b":") + attr_type = line[0:colon_pos].decode("ascii") + + if line[colon_pos:].startswith(b"::"): + attr_value = base64.decodebytes(line[colon_pos + 2 :]) + elif line[colon_pos:].startswith(b":<"): + url = line[colon_pos + 2 :].strip() + attr_value = b"" + if self._process_url_schemes: + u = urlparse(url) + if u[0] in self._process_url_schemes: + attr_value = urlopen(url.decode("ascii")).read() + else: + attr_value = line[colon_pos + 1 :].strip() + + return self._decode_value(attr_type, attr_value) + + def _error(self, msg): + if self._strict: + raise ValueError(msg) + else: + log.warning(msg) + + def _check_dn(self, dn, attr_value): + """Check dn attribute for issues.""" + if dn is not None: + self._error("Two lines starting with dn: in one record.") + if not is_dn(attr_value): + self._error( + f"No valid string-representation of distinguished name {attr_value}." + ) + + def _check_changetype(self, dn, changetype, attr_value): + """Check changetype attribute for issues.""" + if dn is None: + self._error("Read changetype: before getting valid dn: line.") + if changetype is not None: + self._error("Two lines starting with changetype: in one record.") + if attr_value not in CHANGE_TYPES: + self._error(f"changetype value {attr_value} is invalid.") + + def _parse_entry_record(self, lines: list[bytes]) -> tuple[Optional[str], dict]: + """Parse a single entry record from a list of lines.""" + dn = None + entry: dict = OrderedDict() + + for line in lines: + attr_type, attr_value = self._parse_attr(line) + + if attr_type == "version" and dn is None: + continue # version = 1 + + if attr_type == "dn": + self._check_dn(dn, attr_value) + dn = attr_value + continue + + if dn is None: + self._error( + f"First line of record does not start with 'dn:': {attr_type}" + ) + continue + + if attr_type.lower() in self._ignored_attr_types: + continue + + if attr_type in entry: + entry[attr_type].append(attr_value) + else: + entry[attr_type] = [attr_value] + + return dn, entry diff --git a/packages/python3-nethsec/src/nethsec/mwan/__init__.py b/packages/python3-nethsec/src/nethsec/mwan/__init__.py new file mode 100644 index 000000000..b5585826f --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/mwan/__init__.py @@ -0,0 +1,698 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Mwan utilities +""" + + +import json +import subprocess + +import uci +from euci import EUci + +from nethsec import utils, objects +from nethsec.utils import ValidationError + +def _is_valid_src(e_uci: EUci, database_id: str): + """ + Validate the given object for source. + Source objects can be only: + - dhcp reservation + - dns domain + - vpn user + - a singleton host set + + Args: + e_uci: EUci instance + database_id: id of object + + Returns: + True if object is valid, False otherwise + """ + if objects.is_host_set(e_uci, database_id): + return objects.is_singleton_host_set(e_uci, database_id, allow_cidr=True) + + return objects.is_host(e_uci, database_id) or objects.is_domain(e_uci, database_id) or objects.is_vpn_user(e_uci, database_id) + +def _is_valid_dst(e_uci: EUci, database_id: str): + """ + Validate the given object for destination. + Destination objects can be only: + - dhcp reservation + - dns domain + - vpn user + - a singleton host set + + Args: + e_uci: EUci instance + database_id: id of object + + Returns: + True if object is valid, False otherwise + """ + if objects.is_host_set(e_uci, database_id): + return objects.is_singleton_host_set(e_uci, database_id, allow_cidr=True) + + return objects.is_host(e_uci, database_id) or objects.is_domain(e_uci, database_id) or objects.is_vpn_user(e_uci, database_id) + +def __generate_metric(e_uci: EUci) -> int: + """ + Generates a metric for an interface. + + Args: + e_uci: EUci instance + + Returns: + first metric that is not present in interface_metrics + """ + next_metric = 20 + for interface in utils.get_all_by_type(e_uci, 'network', 'interface').values(): + if 'metric' in interface: + next_metric = max(next_metric, int(interface['metric'])) + return next_metric + 10 + + +def __set_interface_defaults(e_uci: EUci, name: str): + default_interface_config = e_uci.get_all('ns-api', 'defaults_mwan') + e_uci.set('mwan3', name, 'initial_state', default_interface_config['initial_state']) + e_uci.set('mwan3', name, 'family', default_interface_config['protocol']) + e_uci.set('mwan3', name, 'track_ip', default_interface_config['track_ip']) + e_uci.set('mwan3', name, 'track_method', default_interface_config['tracking_method']) + e_uci.set('mwan3', name, 'reliability', default_interface_config['tracking_reliability']) + e_uci.set('mwan3', name, 'count', default_interface_config['ping_count']) + e_uci.set('mwan3', name, 'size', default_interface_config['ping_size']) + e_uci.set('mwan3', name, 'max_ttl', default_interface_config['ping_max_ttl']) + e_uci.set('mwan3', name, 'timeout', default_interface_config['ping_timeout']) + e_uci.set('mwan3', name, 'interval', default_interface_config['ping_interval']) + e_uci.set('mwan3', name, 'failure_interval', default_interface_config['ping_failure_interval']) + e_uci.set('mwan3', name, 'recovery_interval', default_interface_config['ping_recovery_interval']) + e_uci.set('mwan3', name, 'down', default_interface_config['interface_down_threshold']) + e_uci.set('mwan3', name, 'up', default_interface_config['interface_up_threshold']) + + +def __store_interface(e_uci: EUci, name: str) -> tuple[bool, bool]: + """ + Stores interface configuration for mwan3 and network, not suited to be used outside store_policy. + + Args: + e_uci: EUci instance + name: name of interface + + Returns: + tuple of booleans, first one indicates if mwan interface was created, second one indicates if metric was added + to network interface + + Raises: + ValidationError: if interface name is not defined in /etc/config/network + """ + # checking if interface is configured + try: + e_uci.get('network', name) + except uci.UciExceptionNotFound: + raise ValidationError('name', 'invalid', name) + + created_interface = False + # if no interface with name exists, create one with defaults + if e_uci.get('mwan3', name, default=None) is None: + created_interface = True + # fetch default configuration and set interface + e_uci.set('mwan3', name, 'interface') + e_uci.set('mwan3', name, 'enabled', '1') + __set_interface_defaults(e_uci, name) + + added_metric = False + # avoid adding metric if already present + if e_uci.get('network', name, 'metric', default=None) is None: + added_metric = True + # generate metric + metric = __generate_metric(e_uci) + # configure metric for interface + e_uci.set('network', name, 'metric', metric) + return created_interface, added_metric + + +def __store_member(e_uci: EUci, interface_name: str, metric: int, weight: int) -> tuple[str, bool]: + """ + Stores member configuration for mwan3, not suited to be used outside store_policy. + + Args: + e_uci: EUci instance + interface_name: name of interface to link the member to + metric: metric of the member + weight: weight of the member + + Returns: + tuple of string and boolean, first one is the generated name of the member, second one indicates if the member + was created + """ + member_config_name = utils.get_id(f'{interface_name}_M{metric}_W{weight}') + changed = False + if e_uci.get('mwan3', member_config_name, default=None) is None: + changed = True + e_uci.set('mwan3', member_config_name, 'member') + e_uci.set('mwan3', member_config_name, 'interface', interface_name) + e_uci.set('mwan3', member_config_name, 'metric', metric) + e_uci.set('mwan3', member_config_name, 'weight', weight) + return member_config_name, changed + + +def store_rule(e_uci: EUci, name: str, policy: str, protocol: str = None, + source_address: str = None, source_port: str = None, + destination_address: str = None, destination_port: str = None, sticky: bool = False, + ns_src: str = None, ns_dst: str = None) -> str: + """ + Stores a rule for mwan3 + + Args: + e_uci: EUci instance + name: name of the rule, must be unique + policy: policy to use for the rule, must be already set + protocol: must be one of tcp, udp, icmp or all, defaults to 'all' + source_address: source addresses to match + source_port: source ports to match or range + destination_address: destination addresses to match + destination_port: destination ports to match or range + sticky: whether to use sticky connections + ns_src: source object, it overrides source_address + ns_dst: destination object, it overrides destination_address + + Returns: + name of the rule created + + Raises: + ValidationError if name is not unique, policy is not valid or length get_id > 15 + """ + # check if the rule length is not more than 15 characters: 12 from user and 3 from get_id prefix + if len(name) > 12: + # get_id add 3 more chars (ns_) to the name + raise ValidationError('name', 'length_12_max', name) + rule_config_name = utils.get_id(name.lower(), 15) + rules = utils.get_all_by_type(e_uci, 'mwan3', 'rule').keys() + if e_uci.get('mwan3', rule_config_name, default=None) is not None: + raise ValidationError('name', 'unique', name) + if e_uci.get('mwan3', policy, default=None) is None: + raise ValidationError('policy', 'invalid', policy) + if ns_src and not _is_valid_src(e_uci, ns_src): + raise ValidationError('ns_src', 'invalid_object', ns_src) + if ns_dst and not _is_valid_dst(e_uci, ns_dst): + raise ValidationError('ns_dst', 'invalid_object', ns_dst) + + e_uci.set('mwan3', rule_config_name, 'rule') + e_uci.set('mwan3', rule_config_name, 'label', name) + e_uci.set('mwan3', rule_config_name, 'use_policy', policy) + # test if sticky is True or False, if not raise an error + if type(sticky) is not bool: + raise ValidationError('sticky', 'sticky_not_valid', sticky) + e_uci.set('mwan3', rule_config_name, 'sticky', sticky) + if protocol is not None: + e_uci.set('mwan3', rule_config_name, 'proto', protocol) + if source_address is not None: + e_uci.set('mwan3', rule_config_name, 'src_ip', source_address) + if source_port is not None: + e_uci.set('mwan3', rule_config_name, 'src_port', source_port.replace('-', ':')) + if destination_address is not None: + e_uci.set('mwan3', rule_config_name, 'dest_ip', destination_address) + if destination_port is not None: + e_uci.set('mwan3', rule_config_name, 'dest_port', destination_port.replace('-', ':')) + if ns_src is not None: + e_uci.set('mwan3', rule_config_name, 'ns_src', ns_src) + if ns_dst is not None: + e_uci.set('mwan3', rule_config_name, 'ns_dst', ns_dst) + + order_rules(e_uci, [rule_config_name] + list(rules)) + update_rules(e_uci) # update rules with objects and save mwan3 config + return f'mwan3.{rule_config_name}' + + +def store_policy(e_uci: EUci, name: str, interfaces: list[dict]) -> list[str]: + """ + Stores a policy for mwan3, takes care of creating interfaces and members. + + Args: + e_uci: EUci instance + name: name of policy + interfaces: list of interfaces to add to policy, must have name, metric and weight fields + + Returns: + list of changed configuration + + Raises: + ValidationError: if name is not unique or length or get_id > 15 + """ + changed_config = [] + if len(name) > 12: + # get_id add 3 more chars (ns_) to the name + raise ValidationError('name', 'length_12_max', name) + # generate policy name + policy_config_name = utils.get_id(name.lower(), 15) + # test length of policy name + # make sure name is not something that already exists + if e_uci.get('mwan3', policy_config_name, default=None) is not None: + raise ValidationError('name', 'unique', name) + # generate policy config with corresponding name + e_uci.set('mwan3', policy_config_name, 'policy') + e_uci.set('mwan3', policy_config_name, 'label', name) + changed_config.append(f'mwan3.{policy_config_name}') + + member_names = __add_interfaces(e_uci, interfaces, changed_config) + + e_uci.set('mwan3', policy_config_name, 'use_member', member_names) + + if len(utils.get_all_by_type(e_uci, 'mwan3', 'rule')) == 0: + changed_config.append(store_rule(e_uci, 'Default Rule', policy_config_name)) + + # default policy must have a last_resort field + if len(utils.get_all_by_type(e_uci, 'mwan3', 'policy')) == 1: + e_uci.set('mwan3', policy_config_name, 'last_resort', 'default') + + e_uci.save('mwan3') + e_uci.save('network') + return changed_config + + +def __fetch_interface_status(interface_name: str) -> str: + try: + output = (subprocess.run([ + 'ubus', + 'call', + 'mwan3', + 'status', + '{"section": "interfaces"}' + ], capture_output=True, check=True) + .stdout.decode('utf-8')) + decoded_output = json.JSONDecoder().decode(output) + return decoded_output['interfaces'][interface_name]['status'] + except: + return 'unknown' + + +def __parse_member(e_uci: EUci, member_name: str) -> dict: + """ + Parses a member configuration and returns formatted data. + + Args: + e_uci: EUci instance + member_name: member name + + Returns: + dict with member data + """ + member_data = e_uci.get_all('mwan3', member_name) + return { + 'name': member_name, + 'interface': member_data['interface'], + 'metric': member_data['metric'], + 'weight': member_data['weight'], + 'status': __fetch_interface_status(member_data['interface']) + } + + +def index_policies(e_uci: EUci) -> list[dict]: + """ + Returns a list of policies with their members, interfaces and metrics/weights. + + Args: + e_uci: EUci instance + + Returns: + list of dicts with policy data + """ + data = [] + policies = utils.get_all_by_type(e_uci, 'mwan3', 'policy') + # iterate over policies + for policy_name in policies.keys(): + policy = policies[policy_name] + policy_data = { + 'name': policy_name, + } + # add label only if present + if 'label' in policy: + policy_data['label'] = policy['label'] + + # add members + members = [] + if 'use_member' in policy: + policy_data['members'] = {} + for member in policy['use_member']: + members.append(__parse_member(e_uci, member)) + + # infer policy type by metrics + metrics = [int(member['metric']) for member in members] + if len(metrics) == 1: + policy_data['type'] = 'custom' + elif all(metric == metrics[0] for metric in metrics): + policy_data['type'] = 'balance' + elif all(metrics.index(metric) == key for key, metric in enumerate(metrics)): + policy_data['type'] = 'backup' + else: + policy_data['type'] = 'custom' + + unique_metrics = list(set(metrics)) + for unique_metric in unique_metrics: + policy_data['members'][unique_metric] = list(filter(lambda x: x['metric'] == str(unique_metric), members)) + + # append policy to data + data.append(policy_data) + return data + + +def __add_interfaces(e_uci: EUci, interfaces: list[dict], changed_config: list[str] = None) -> list[str]: + """ + Add interfaces to policy, takes care of creating interfaces and members. + + Args: + e_uci: euci instance + interfaces: list of interfaces to add + changed_config: array of changed configuration + + Returns: + list of member names added to policy + + Raises: + ValidationError: if interface name is not defined in /etc/config/network + """ + if changed_config is None: + changed_config = list() + member_names: list[str] = [] + for interface in interfaces: + try: + added_mwan_interface, updated_interface = __store_interface(e_uci, interface['name']) + except ValidationError: + raise ValidationError('interfaces', 'invalid', interface['name']) + if added_mwan_interface: + changed_config.append(f'mwan3.{interface["name"]}') + if updated_interface: + changed_config.append(f'network.{interface["name"]}') + + member_config_name, member_created = __store_member(e_uci, + interface['name'], + interface['metric'], + interface['weight']) + member_names.append(member_config_name) + if member_created: + changed_config.append(f'mwan3.{member_config_name}') + + return member_names + + +def edit_policy(e_uci: EUci, name: str, label: str, interfaces: list[dict]) -> list[str]: + """ + Edits a mwan3 policy. + + Args: + e_uci: euci instance + name: name of policy to edit + label: policy label + interfaces: dict of interfaces to add to policy + + Returns: + list of changed mwan3 entries + + Raises: + ValidationError: if name is not valid + """ + if e_uci.get('mwan3', name, default=None) is None: + raise ValidationError('name', 'invalid', name) + changed_config = [] + if label != e_uci.get_all('mwan3', name)['label']: + e_uci.set('mwan3', name, 'label', label) + changed_config.append(f'mwan3.{name}') + + member_names = __add_interfaces(e_uci, interfaces, changed_config) + + e_uci.set('mwan3', name, 'use_member', member_names) + + e_uci.save('mwan3') + e_uci.save('network') + return changed_config + + +def delete_policy(e_uci: EUci, name: str) -> list[str]: + """ + Deletes a mwan3 policy. + + Args: + e_uci: euci instance + name: name of policy to delete + + Returns: + list of deleted mwan3 entries + """ + if e_uci.get('mwan3', name, default=None) is None: + raise ValidationError('name', 'invalid', name) + e_uci.delete('mwan3', name) + e_uci.save('mwan3') + return [f'mwan3.{name}'] + + +def index_rules(e_uci: EUci) -> list[dict]: + """ + Returns a list of rules with their policies. + + Args: + e_uci: euci instance + + Returns: + parsed list of rules with assigned policy + """ + data = [] + rules = utils.get_all_by_type(e_uci, 'mwan3', 'rule') + for rule_key in rules.keys(): + rule_data = {} + rule_value = rules[rule_key] + rule_data['name'] = rule_key + rule_data['policy'] = {} + rule_data['policy']['name'] = rule_value['use_policy'] + if e_uci.get('mwan3', rule_value['use_policy'], default=None) is not None: + rule_data['policy']['label'] = utils.get_all_by_type(e_uci, 'mwan3', 'policy')[rule_value['use_policy']].get('label') + if 'label' in rule_value: + rule_data['label'] = rule_value['label'] + if 'proto' in rule_value: + rule_data['protocol'] = rule_value['proto'] + if 'src_ip' in rule_value: + rule_data['source_address'] = rule_value['src_ip'] + if 'src_port' in rule_value: + rule_data['source_port'] = rule_value['src_port'].replace(':', '-') + if 'dest_ip' in rule_value: + rule_data['destination_address'] = rule_value['dest_ip'] + if 'dest_port' in rule_value: + rule_data['destination_port'] = rule_value['dest_port'].replace(':', '-') + if 'sticky' in rule_value: + rule_data['sticky'] = rule_value['sticky'] == '1' + if 'ns_src' in rule_value: + rule_data['ns_src'] = rule_value['ns_src'] + if 'ns_dst' in rule_value: + rule_data['ns_dst'] = rule_value['ns_dst'] + + data.append(rule_data) + return data + + +def order_rules(e_uci: EUci, rules: list[str]) -> list[str]: + """ + Orders mwan3 rules, moves everything else but rules to the end of the list. + + Args: + e_uci: euci instance + rules: which order to put rules + + Returns: + list of ordered mwan3 entries + + Raises: + ValidationError: if a rule is not present in /etc/config/mwan3 + """ + for rule in utils.get_all_by_type(e_uci, 'mwan3', 'rule').keys(): + if rule not in rules: + raise ValidationError('rules', 'missing', rule) + + order: list[str] = [] + + for key in e_uci.get_all('mwan3').keys(): + if key not in rules: + order.append(key) + + order.extend(rules) + + subprocess.run([ + 'ubus', + 'call', + 'uci', + 'order', + json.dumps({ + 'config': 'mwan3', + 'sections': order, + }) + ]) + + e_uci.save('mwan3') + return order + + +def delete_rule(e_uci: EUci, name: str): + """ + Deletes a mwan3 rule. + + Args: + e_uci: euci instance + name: rule name to delete + + Returns: + name of rule deleted + """ + if e_uci.get('mwan3', name, default=None) is None: + raise ValidationError('name', 'invalid', name) + + e_uci.delete('mwan3', name) + e_uci.save('mwan3') + return f'mwan3.{name}' + + +def edit_rule(e_uci: EUci, name: str, policy: str, label: str, protocol: str = None, + source_address: str = None, source_port: str = None, + destination_address: str = None, destination_port: str = None, sticky: bool = False, + ns_src: str = None, ns_dst: str = None): + """ + Edits a mwan3 rule. + + Args: + e_uci: EUci instance + name: rule name + policy: policy to use for the rule + label: rule label + protocol: protocol in which the rule applies + source_address: CIDR notation of source address + source_port: port or port range + destination_address: CIDR notation of destination address + destination_port: port or port range + sticky: whether to use sticky connections + ns_src: source object, it overrides source_address + ns_dst: destination object, it overrides destination_address + + Raises: + ValidationError: if name is not valid or policy is not valid + """ + if e_uci.get('mwan3', name, default=None) is None: + raise ValidationError('name', 'invalid', name) + + if e_uci.get('mwan3', policy, default=None) is None: + raise ValidationError('policy', 'invalid', policy) + if ns_src and not _is_valid_src(e_uci, ns_src): + raise ValidationError('ns_src', 'invalid_object', ns_src) + if ns_dst and not _is_valid_dst(e_uci, ns_dst): + raise ValidationError('ns_dst', 'invalid_object', ns_dst) + e_uci.set('mwan3', name, 'use_policy', policy) + e_uci.set('mwan3', name, 'label', label) + # test if sticky is True of False, if not raise an error + if type(sticky) is not bool: + raise ValidationError('sticky', 'sticky_not_valid', sticky) + e_uci.set('mwan3', name, 'sticky', sticky) + if protocol is not None: + e_uci.set('mwan3', name, 'proto', protocol) + if protocol != 'tcp' and protocol != 'udp': + e_uci.delete('mwan3', name, 'src_port') + e_uci.delete('mwan3', name, 'dest_port') + else: + if destination_port is not None: + e_uci.set('mwan3', name, 'dest_port', destination_port.replace('-', ':')) + if source_port is not None: + e_uci.set('mwan3', name, 'src_port', source_port.replace('-', ':')) + if source_address is not None: + e_uci.set('mwan3', name, 'src_ip', source_address) + else: + e_uci.delete('mwan3', name, 'src_ip') + if destination_address is not None: + e_uci.set('mwan3', name, 'dest_ip', destination_address) + else: + e_uci.delete('mwan3', name, 'dest_ip') + if ns_src is not None: + e_uci.set('mwan3', name, 'ns_src', ns_src) + else: + e_uci.delete('mwan3', name, 'ns_src') + if ns_dst is not None: + e_uci.set('mwan3', name, 'ns_dst', ns_dst) + else: + e_uci.delete('mwan3', name, 'ns_dst') + update_rules(e_uci) # update rules with objects and save mwan3 config + return f'mwan3.{name}' + + +def clear_config(e_uci: EUci): + """ + Clears mwan3 configuration. + + Args: + e_uci: euci instance + """ + for entry in e_uci.get_all('mwan3'): + if entry != 'globals': + e_uci.delete('mwan3', entry) + + e_uci.save('mwan3') + + +def set_default_config(e_uci: EUci, track_ip: list[str], ping_timeout: int, ping_interval: int, + ping_failure_interval: int, interface_down_threshold: int, interface_up_threshold: int): + """ + Sets default configuration for mwan3. + + Args: + e_uci: euci instance + track_ip: list of IPs to track + ping_timeout: timeout for ping + ping_interval: interval between pings + ping_failure_interval: interval between failed pings + interface_down_threshold: threshold for interface down + interface_up_threshold: threshold for interface up + """ + e_uci.set('ns-api', 'defaults_mwan', 'track_ip', track_ip) + e_uci.set('ns-api', 'defaults_mwan', 'ping_timeout', ping_timeout) + e_uci.set('ns-api', 'defaults_mwan', 'ping_interval', ping_interval) + e_uci.set('ns-api', 'defaults_mwan', 'ping_failure_interval', ping_failure_interval) + e_uci.set('ns-api', 'defaults_mwan', 'interface_down_threshold', interface_down_threshold) + e_uci.set('ns-api', 'defaults_mwan', 'interface_up_threshold', interface_up_threshold) + + for interface in utils.get_all_by_type(e_uci, 'mwan3', 'interface'): + __set_interface_defaults(e_uci, interface) + + e_uci.save('ns-api') + e_uci.save('mwan3') + + +def get_default_config(e_uci: EUci) -> dict: + """ + Returns default configuration for mwan3. + + Args: + e_uci: euci instance + + Returns: + dict with default configuration + """ + return e_uci.get_all('ns-api', 'defaults_mwan') + + +def update_rules(e_uci: EUci): + """ + Updates mwan3 rules with objects addresses + + Args: + e_uci: euci instance + """ + for rule in utils.get_all_by_type(e_uci, 'mwan3', 'rule'): + ns_src = e_uci.get('mwan3', rule, 'ns_src', default=None) + ns_dst = e_uci.get('mwan3', rule, 'ns_dst', default=None) + # both ns_src and ns_dst should be a singleton + if ns_src: + e_uci.set('mwan3', rule, 'src_ip', objects.get_object_ip(e_uci, ns_src)) + if ns_dst: + # domain sets are not supported because mwan3 ipset functionality is broken on 23.05. + e_uci.set('mwan3', rule, 'dest_ip', objects.get_object_ip(e_uci, ns_dst)) + + e_uci.save('mwan3') diff --git a/packages/python3-nethsec/src/nethsec/objects/__init__.py b/packages/python3-nethsec/src/nethsec/objects/__init__.py new file mode 100644 index 000000000..ece401749 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/objects/__init__.py @@ -0,0 +1,884 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +Objects utilities +''' +import ipaddress +import json +import os +import subprocess + +from euci import EUci + +from nethsec import utils, firewall + +# Generic + +def is_object_id(id): + """ + Check if an id is an object id. + + Args: + id: id to check + + Returns: + True if id is an object id, False otherwise + """ + if not id or not isinstance(id, str): + return False + return id.startswith('objects/') or id.startswith('dhcp/') or id.startswith('users/') + +def object_exists(uci, database_id): + """ + Check if the object exists. + + Args: + database_id: id to check in the form of `/` + + Returns: + True if object exists, False otherwise + """ + try: + database, id = database_id.split('/') + uci.get(database, id) + return True + except: + return False + +def get_object(uci, database_id): + """ + Get object from objects config. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + object from config or None if not found + """ + try: + database, id = database_id.split('/') + return uci.get_all(database, id) + except: + return None + +def is_used_object(uci, database_id): + """ + Check if an object is used in: + - firewall config + - mwan3 config + - dpi config + - existing host set + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + A tuple with: + - True if domain set is used in firewall config, False otherwise + - a list of firewall sections where domain set is used + """ + matches = [] + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_src', default=None) == database_id or uci.get('firewall', section, 'ns_dst', default=None) == database_id: + matches.append(f'firewall/{section}') + for section in uci.get_all("mwan3"): + if uci.get('mwan3', section, 'ns_src', default=None) == database_id or uci.get('mwan3', section, 'ns_dst', default=None) == database_id: + matches.append(f'mwan3/{section}') + for section in uci.get_all("dpi"): + if uci.get('dpi', section, 'source', default=None) == database_id: + matches.append(f'dpi/{section}') + for section in uci.get_all("objects"): + try: + ips = uci.get_all('objects', section, 'ipaddr') + if database_id in ips: + matches.append(f'objects/{section}') + except: + continue + return len(matches) > 0, matches + +def get_object_ips(uci, database_id): + """ + Get all IP addresses from an object. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + a list of unique IP addresses from the object + """ + ips = [] + if not database_id: + return ips + + obj = get_object(uci, database_id) + database, id = database_id.split('/') + + if not obj: + return ips + + if database == 'dhcp': + ip = obj.get('ip') + if ip: + ips.append(ip) + elif database == 'users': + openvpn_ipaddr = obj.get('openvpn_ipaddr') + if openvpn_ipaddr: + ips.append(openvpn_ipaddr) + elif database == 'objects': + ipaddr = obj.get('ipaddr') + if ipaddr: + for ip in ipaddr: + if is_object_id(ip): + ips.extend(get_object_ips(uci, ip)) + else: + ips.append(ip) + + return list(set(ips)) # Convert the list to a set to remove duplicates, then convert it back to a list + +def get_object_ip(uci, database_id): + """ + Get the first IP address from an object. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + the first IP address from the object + """ + ips = get_object_ips(uci, database_id) + if ips: + return ips[0] + return None + +# Domain set + +def is_domain_set(uci, database_id): + """ + Check if an object is a domain set. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + True if object is a domain set, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database =="objects" and obj_type == "domain" + except: + return False + +def is_used_domain_set(uci, id): + """ + Check if domain set is used in firewall config. + + Args: + uci: EUci pointer + id: id of domain set + + Returns: + A tuple with: + - True if domain set is used in firewall config, False otherwise + - a list of firewall sections where domain set is used + """ + return is_used_object(uci, f'objects/{id}') + +def get_domain_set_ipsets(uci, id): + """ + Get ipsets linked to domain set. + + Args: + uci: EUci pointer + id: id of domain set + + Returns: + a dictionary with + - `firewall`: the ipset id linked to domain set from firewall config + - `dhcp`: the ipset id linked to domain set from dhcp config + """ + ipsets = {"firewall": None, "dhcp": None} + for section in utils.get_all_by_type(uci, "firewall", "ipset"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + ipsets["firewall"] = uci.get("firewall", section, "name", default='') + break + for section in utils.get_all_by_type(uci, "dhcp", "ipset"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + ipsets["dhcp"] = section + break + return ipsets + +def get_domain_set_total_chars(name: str, unique_domains: list[str]): + """ + Calculate the total number of characters the given domain set will use in the dnsmasq configuration file. + This file supports a maximum of 1024 characters per line. + + Example syntax of a domain set in dnsmasq configuration file: + nftset=/example1.org/example2.org/example3.org/.../4#inet#fw4#domainSetName + + Args: + name: name of domain set + domains: a list of valid DNS names + + Returns: + the number of characters + """ + total_chars = 8 + 11 # leading "nftset=/" and trailing "4#inet#fw4#" + + for domain in unique_domains: + total_chars += len(domain) + 1 # +1 for trailing "/" + + total_chars += len(name) # domain set name + return total_chars + +def add_domain_set(uci, name: str, family: str, domains: list[str], timeout: int = 660) -> str: + """ + Add domain set to objects config. + + Args: + uci: EUci pointer + name: name of domain set + family: can be `ipv4` or `ipv6` + domains: a list of valid DNS names + timeout: the timeout in seconds for the DNS resolution, default is `660` seconds + + Returns: + id of domain set config that was added + """ + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + # check name contains only number and letters + if not name.isalnum(): + raise utils.ValidationError('name', 'invalid_name', name) + if family not in ['ipv4', 'ipv6']: + raise utils.ValidationError('family', 'invalid_family', family) + if timeout < 0: + raise utils.ValidationError('timeout', 'invalid_timeout', timeout) + + # remove duplicated domains + unique_domains = list(dict.fromkeys(domains)) + if get_domain_set_total_chars(name, unique_domains) > 1024 - 10: # -10 chars for safety + raise utils.ValidationError('domain', 'total_chars_exceeded') + + id = utils.get_random_id() + uci.set('objects', id, 'domain') + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'timeout', timeout) + uci.set('objects', id, 'domain', unique_domains) + uci.save('objects') + + # create ipset inside dhcp config + ipset = utils.get_random_id() + uci.set('dhcp', ipset, 'ipset') + uci.set('dhcp', ipset, 'name', [name]) + uci.set('dhcp', ipset, 'domain', unique_domains) + uci.set('dhcp', ipset, 'table_family', 'inet') + uci.set('dhcp', ipset, 'ns_link', f'objects/{id}') + uci.save('dhcp') + + # create ipset inside firewall config + ipset = utils.get_random_id() + uci.set('firewall', ipset, 'ipset') + uci.set('firewall', ipset, 'name', name) + uci.set('firewall', ipset, 'family', family) + uci.set('firewall', ipset, 'timeout', timeout) + uci.set('firewall', ipset, 'counters', '1') + uci.set('firewall', ipset, 'match', 'ip') + uci.set('firewall', ipset, 'ns_link', f'objects/{id}') + uci.save('firewall') + return id + +def edit_domain_set(uci, id: str, name: str, family: str, domains: list[str], timeout: int = 660) -> str: + """ + Edit domain set in objects config. + + Args: + uci: EUci pointer + id: id of domain set to edit + name: name of domain set + family: can be `ipv4` or `ipv6` + domains: a list of valid DNS names + timeout: the timeout in seconds for the DNS resolution, default is `660` seconds + + Returns: + id of domain set config that was edited + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "domain_set_does_not_exists", id) + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + if family not in ['ipv4', 'ipv6']: + raise utils.ValidationError('family', 'invalid_family', family) + if timeout < 0: + raise utils.ValidationError('timeout', 'invalid_timeout', timeout) + + # remove duplicated domains + unique_domains = list(dict.fromkeys(domains)) + if get_domain_set_total_chars(name, unique_domains) > 1024 - 10: # -10 chars for safety + raise utils.ValidationError('domain', 'total_chars_exceeded') + + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'timeout', timeout) + uci.set('objects', id, 'domain', unique_domains) + uci.save('objects') + + # update ipset inside dhcp config + for section in uci.get_all("dhcp"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + uci.set('dhcp', section, 'name', [name]) + uci.set('dhcp', section, 'domain', unique_domains) + uci.set('dhcp', section, 'ns_tag', ['automated']) + uci.save('dhcp') + break + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + uci.set('firewall', section, 'name', name) + uci.set('firewall', section, 'family', family) + uci.set('firewall', section, 'timeout', timeout) + uci.set('firewall', section, 'ns_tag', ['automated']) + uci.save('firewall') + break + return id + +def delete_domain_set(uci, id: str) -> str: + """ + Delete domain set from objects config. + + Args: + uci: EUci pointer + id: id of domain set to delete + + Returns: + name of domain set config that was deleted + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "domain_set_does_not_exists", id) + used, matches = is_used_domain_set(uci, id) + if used: + raise utils.ValidationError("id", "domain_set_is_used", matches) + uci.delete('objects', id) + uci.save('objects') + for section in uci.get_all("dhcp"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + uci.delete('dhcp', section) + uci.save('dhcp') + break + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + uci.delete('firewall', section) + uci.save('firewall') + break + return id + +def list_domain_sets(uci, used_info = True) -> list: + """ + Get all domain sets from objects config + + Args: + uci: EUci pointer + used_info: include used and matches info + + Returns: + a list of all domain sets + """ + sets = [] + for section in uci.get_all("objects"): + if uci.get('objects', section) == 'domain': + rule = uci.get_all('objects', section) + rule['id'] = section + rule['type'] = 'domain_set' + rule['subtype'] = 'domain_set' + if used_info: + used, matches = is_used_domain_set(uci, section) + rule['used'] = used + rule['matches'] = matches + sets.append(rule) + return sets + +# Host set + +def _has_loop(uci, id, ipaddr, depth=0): + if depth > 2: + return True + if is_object_id(ipaddr): + if ipaddr == id: + return True + obj = get_object(uci, ipaddr) + if obj: + for ip in obj.get('ipaddr', []): + if _has_loop(uci, id, ip, depth + 1): + return True + return False + +def _validate_host_set_ipaddr(uci, id, ipaddr: str, family: str): + if is_object_id(ipaddr): + if not object_exists(uci, ipaddr): + raise utils.ValidationError('ipaddr', 'object_does_not_exists', ipaddr) + else: + if id and is_host_set(uci, id): + # check loop + if _has_loop(uci, id, ipaddr): + raise utils.ValidationError('ipaddr', 'loop_detected', ipaddr) + return # validation is ok + + if family == 'ipv4': + return _validate_host_set_ipaddr_v4(ipaddr) + elif family == 'ipv6': + return _validate_host_set_ipaddr_v6(ipaddr) + +def _validate_host_set_ipaddr_v4(ipaddr: str): + if '/' in ipaddr: + # validate CIDR + try: + ipaddress.IPv4Network(ipaddr) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + elif '-' in ipaddr: + start, end = ipaddr.split('-') + try: + ipaddress.IPv4Address(start) + ipaddress.IPv4Address(end) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + else: + # validate IPv4 + try: + ipaddress.IPv4Address(ipaddr) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + return True + +def _validate_host_set_ipaddr_v6(ipaddr: str): + if '/' in ipaddr: + # validate CIDR + try: + ipaddress.IPv6Network(ipaddr) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + elif '-' in ipaddr: + start, end = ipaddr.split('-') + try: + ipaddress.IPv6Address(start) + ipaddress.IPv6Address(end) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + else: + # validate IPv6 + try: + ipaddress.IPv6Address(ipaddr) + except: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + return True + +def add_host_set(uci, name: str, family: str, ipaddrs: list[str]) -> str: + """ + Add host set to objects config. + + Args: + uci: EUci pointer + name: name of host set + family: can be `ipv4` or `ipv6` + ipaddrs: a list of IP addresses + + Returns: + id of host set config that was added + """ + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + # check name contains only number and letters + if not name.isalnum(): + raise utils.ValidationError('name', 'invalid_name', name) + for ipaddr in ipaddrs: + _validate_host_set_ipaddr(uci, '', ipaddr, family) + id = utils.get_random_id() + uci.set('objects', id, 'host') + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'ipaddr', ipaddrs) + uci.save('objects') + return id + +def edit_host_set(uci, id: str, name: str, family: str, ipaddrs: list[str]) -> str: + """ + Edit host set in objects config. + + Args: + uci: EUci pointer + id: id of host set to edit + name: name of host set + family: can be `ipv4` or `ipv6` + ipaddrs: a list of IP addresses + + Returns: + id of host set config that was edited + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "host_set_does_not_exists", id) + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + for ipaddr in ipaddrs: + _validate_host_set_ipaddr(uci, f'objects/{id}', ipaddr, family) + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'ipaddr', ipaddrs) + uci.save('objects') + return id + +def delete_host_set(uci, id: str) -> str: + """ + Delete host set from objects config. + + Args: + uci: EUci pointer + id: id of host set to delete + + Returns: + name of host set config that was deleted + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "host_set_does_not_exists", id) + used, matches = is_used_host_set(uci, id) + if used: + raise utils.ValidationError("id", "host_set_is_used", matches) + uci.delete('objects', id) + uci.save('objects') + return id + +def is_used_host_set(uci, id): + """ + Check if host set is used in firewall config. + + Args: + uci: EUci pointer + id: id of host set + + Returns: + A tuple with: + - True if host set is used in firewall config, False otherwise + - a list of firewall sections where host set is used + """ + return is_used_object(uci, f'objects/{id}') + +def list_host_sets(uci, used_info = True) -> list: + """ + Get all host sets from objects config + + Args: + uci: EUci pointer + used_info: include used and matches info + + Returns: + a list of all host sets + """ + sets = [] + for section in uci.get_all("objects"): + if uci.get('objects', section) == 'host': + rule = uci.get_all('objects', section) + rule['id'] = section + rule['singleton'] = is_singleton_host_set(uci, f'objects/{section}') + if rule['singleton']: + # set subtype to CIDR, range or host + ip = get_object_ip(uci, f'objects/{section}') + if '/' in ip: + rule['subtype'] = 'cidr' + elif '-' in ip: + rule['subtype'] = 'range' + else: + rule['subtype'] = 'host' + else: + rule['subtype'] = 'host_set' + if used_info: + used, matches = is_used_host_set(uci, section) + rule['used'] = used + rule['matches'] = matches + sets.append(rule) + return sets + +def is_host_set(uci, database_id): + """ + Check if an object is a host set. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + True if object is a host set, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "objects" and obj_type == "host" + except: + return False + +# in mathematical terms, a singleton set is a set with exactly one element +def is_singleton_host_set(uci, database_id, allow_cidr=False): + """ + Check if an object is a host set with a single IP address. + The IP must not be an IP range. + If `allow_cidr` is True, the IP can be in CIDR notation. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + allow_cidr: allow CIDR notation + + Returns: + True if object is a singleton host set, False otherwise + """ + if is_host_set(uci, database_id): + obj = get_object(uci, database_id) + if obj and len(obj.get('ipaddr')) == 1: + ip = obj.get('ipaddr')[0] + if '-' in ip: + return False + if allow_cidr: + return True + else: + return '/' not in ip + return False + +# Host + +def is_host(uci, database_id): + """ + Check if an object is a host. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a host, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "dhcp" and obj_type == "host" + except: + return False + +# Domain + +def is_domain(uci, database_id): + """ + Check if an object is a domain. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a domain, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "dhcp" and obj_type == "domain" + except: + return False + +# VPN user + +def is_vpn_user(uci, database_id): + """ + Check if an object is a VPN user. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a VPN user, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "users" and obj_type == "user" and uci.get(database, id, 'openvpn_ipaddr', default=None) != None + except: + return False + +# API suggestions functions + +# Each element of the list should contain the following fields: +# - `id`: the id of the object +# - `name`: the name of the object +# - `type`: the type of the object +# - `family`: the family of the object (optional) +# If expand flag is set to True, the list should contain all IP addresses of the object + +def list_vpn_users(uci, expand=False, used_info=False): + """ + Get all VPN users from users config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + used_info: include used and matches info + + Returns: + a list of all VPN users + """ + users = [] + for section in uci.get_all("users"): + user = {} + if uci.get('users', section) == 'user' and uci.get('users', section, 'openvpn_ipaddr', default=None) != None: + obj = uci.get_all('users', section) + user['id'] = f"users/{section}" + user['name'] = obj.get('name') + user['type'] = 'vpn_user' + user['subtype'] = 'vpn_user' + user['family'] = 'ipv4' + if expand: + user['ipaddr'] = [obj.get('openvpn_ipaddr')] + if used_info: + used, matches = is_used_object(uci, f'users/{section}') + user['used'] = used + user['matches'] = matches + users.append(user) + return users + +def list_dhcp_static_leases(uci, expand=False, used_info=False): + """ + Get all DHCP static leases from dhcp config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + used_info: include used and matches info + + Returns: + a list of all DHCP static leases + """ + leases = [] + for section in uci.get_all("dhcp"): + lease = {} + if uci.get('dhcp', section) == 'host': + obj = uci.get_all('dhcp', section) + lease['id'] = f"dhcp/{section}" + lease['name'] = obj.get('name') + lease['type'] = 'dhcp_static_lease' + lease['subtype'] = 'dhcp_static_lease' + lease['family'] = 'ipv4' + if expand: + lease['ipaddr'] = [obj.get('ip')] + if used_info: + used, matches = is_used_object(uci, f'dhcp/{section}') + lease['used'] = used + lease['matches'] = matches + leases.append(lease) + return leases + +def list_dns_records(uci, expand=False, used_info=False): + """ + Get all DNS records from dhcp config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + used_info: include used and matches info + + Returns: + a list of all DNS records + """ + records = [] + for section in uci.get_all("dhcp"): + record = {} + if uci.get('dhcp', section) == 'domain': + obj = uci.get_all('dhcp', section) + record['id'] = f"dhcp/{section}" + record['name'] = obj.get('name') + record['type'] = 'dns_record' + record['subtype'] = 'dns_record' + record['family'] = 'ipv4' + if expand: + record['ipaddr'] = [obj.get('ip')] + if used_info: + used, matches = is_used_object(uci, f'dhcp/{section}') + record['used'] = used + record['matches'] = matches + records.append(record) + return records + +def list_objects(uci, include_domain_sets=True, include_host_sets=True, singleton_only=False, expand=False): + """ + Get objects from objects, dhcp, and users config + + Args: + uci: EUci pointer + include_domain_sets: include domain sets in the list + include_host_sets: include host sets in the list + expand: expand the list with all IP addresses of the object + + Returns: + a list of all objects + """ + hsets = [] + dsets = [] + if include_host_sets: + for h in list_host_sets(uci, True): + if singleton_only and not h['singleton']: + continue + h['id'] = f"objects/{h['id']}" + h['type'] = 'host_set' + if not expand: + del[h['ipaddr']] + hsets.append(h) + + if include_domain_sets: + for d in list_domain_sets(uci, True): + d['id'] = f"objects/{d['id']}" + d['type'] = 'domain_set' + d['subtype'] = 'domain_set' + if not expand: + del[d['domain']] + del[d['timeout']] + dsets.append(d) + vpn_users = list_vpn_users(uci, expand, True) + dhcp_static_leases = list_dhcp_static_leases(uci, expand, True) + dns_records = list_dns_records(uci, expand, True) + return hsets + dsets + vpn_users + dhcp_static_leases + dns_records + +def get_info(uci, database_id): + """ + Get the info of the object. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + a dictionary with the following fields: + + - `database`: the database of the object + - `id`: the id of the object + - `name`: the name of the object + - `type`: the type of the object + - `family`: IP family, like `ipv4` or `ipv6` + """ + try: + database, id = database_id.split('/') + type = uci.get(database, id) + name = uci.get(database, id, 'name', default=None) + family = uci.get(database, id, 'family', default='ipv4') + if not name: + name = uci.get(database, id, 'label', default=None) + if not name: + name = id + return {'database': database, 'id': id, 'name': name, 'type': type, 'family': family} + except: + return None + diff --git a/packages/python3-nethsec/src/nethsec/ovpn/__init__.py b/packages/python3-nethsec/src/nethsec/ovpn/__init__.py new file mode 100644 index 000000000..58658d257 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/ovpn/__init__.py @@ -0,0 +1,285 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Library for common VPN functions. +""" + +import json +import ipaddress +import random +import struct +import socket +import subprocess +from nethsec import utils + +def get_local_networks(u): + """ + Return a list of local networks. + + Arguments: + - u -- EUci instance + + Returns: + - a list of local networks + """ + ret = set() + for l in utils.get_all_lan_devices(u): + try: + data = json.loads(subprocess.run(["ip", "--json", "address", "show", "dev", l], capture_output=True, text=True, check=True).stdout) + if len(data) > 0: + for addr in data[0].get('addr_info', []): + if addr.get("local", None) and addr.get("family", None) == "inet": # ipv4 only + # skip bond management address + if addr.get("local").startswith("127"): + continue + net = ipaddress.ip_interface(f'{addr.get("local")}/{addr.get("prefixlen")}').network + ret.add(f'{net}') + except: + continue + return list(ret) + +def get_public_addresses(u): + """ + Return a list of public addresses. + + Arguments: + - u -- EUci instance + + Returns: + - a list of public addresses + """ + ret = {} + for w in utils.get_all_wan_devices(u): + try: + data = json.loads(subprocess.run(["ip", "--json", "address", "show", "dev", w], capture_output=True, text=True, check=True).stdout) + except: + continue + if len(data) > 0: + for addr in data[0].get('addr_info', []): + if addr.get("local", None): + try: + res = subprocess.run([ + "/usr/bin/dig", + "-b", + addr.get("local"), + "+short", + "+time=1", + "myip.opendns.com", + "@resolver1.opendns.com" + ], + capture_output=True, text=True, check=True) + ret[res.stdout.strip()] = 1 + except: + pass + return list(ret.keys()) + +def to_cidr(netmask): + """ + Convert a netmask to CIDR notation. + + Arguments: + - netmask -- netmask in dotted decimal notation + + Returns: + - CIDR notation + + """ + return sum([bin(int(x)).count('1') for x in netmask.split('.')]) + +def to_netmask(prefix): + """ + Convert a CIDR notation to netmask. + + Arguments: + - prefix -- CIDR notation + + Returns: + - netmask in dotted decimal notation + """ + return socket.inet_ntoa(struct.pack(">I", (0xffffffff << (32 - int(prefix))) & 0xffffffff)) + +def generate_random_network(u): + """ + Generate a random network. + + Arguments: + - u -- EUci instance + + Returns: + - a random unused network + """ + network = random_network() + while is_used_network(u, network): + network = random_network() + return network + +def random_network(): + """ + Generate a random private network address. + + Returns: + - a random private network address + """ + return f"10.{random.randint(0, 254)}.{random.randint(0, 254)}.0/24" + +def is_used_network(u, network): + """ + Check if a network is already used by another OpenVPN. + + Arguments: + - u -- EUci instance + - network -- network to check + + Returns: + - True if the network is already used, False otherwise + """ + try: + openvpn = u.get_all('openvpn') + except: + return False + + for v in openvpn: + ifconfig = u.get('openvpn', v, 'ifconfig', default="") + if ifconfig: + ifnet = f'{ifconfig.split(" ")[0]}/24' # assume /24 + if ipaddress.ip_network(ifnet, strict=False) == ipaddress.ip_network(network): + return True + + server = opt2cidr(u.get('openvpn', v, 'server', default="")) + if server and server == network: + return True + + server_bridge = opt2cidr(u.get('openvpn', v, 'server_bridge', default="")) + if server_bridge: + if ipaddress.ip_network(server_bridge, strict=False) == ipaddress.ip_network(network): + return True + return False + +def opt2cidr(opt): + """ + Convert an OpenVPN option to CIDR notation. + + Arguments: + - opt -- OpenVPN option + + Returns: + - CIDR notation + """ + try: + tmp = opt.split(" ") + return f'{tmp[0]}/{to_cidr(tmp[1])}' + except: + return "" + + +def list_cipher(): + """ + Return a list of OpenVPN ciphers. + + Returns: + - a list of OpenVPN ciphers, each element is a dicttionary with the following keys: + - name: cipher name + - description: cipher description (weak or strong) + """ + ret = [] + try: + result = subprocess.run(['/usr/sbin/openvpn', '--show-ciphers'], capture_output=True, text=True, check=True) + output_lines = result.stdout.splitlines() + + for line in output_lines: + if '(' in line: + description = 'weak' + tmp = line.split() + cipher_name = tmp[0] + try: + bits = int(cipher_name.split("-")[1]) + except: + bits = 0 + if bits in [192, 256, 384, 512]: + description = 'strong' + ret.append({'name': cipher_name, 'description': description}) + except: + return {"ciphers": []} + return {"ciphers": ret} + +def list_digest(): + """ + Return a list of OpenVPN digests. + + Returns: + - a list of OpenVPN digests, each element is a dicttionary with the following keys: + - name: digest name + - description: digest description (weak or strong) + """ + ret = [] + try: + result = subprocess.run(['/usr/sbin/openvpn', '--show-digests'], capture_output=True, text=True, check=True) + output_lines = result.stdout.splitlines() + + for line in output_lines: + if 'bit' in line: + description = 'weak' + tmp = line.split() + if tmp[1] in ['224', '256', '384', '512', 'whirlpool']: + description = 'strong' + ret.append({'name': tmp[0], 'description': description}) + except: + return {"digests": []} + return {"digests": ret} + +def generate_random_port(uci, limit_min, limit_max): + ''' + Generate a random port. + + Arguments: + - uci -- EUci instance + - limit_min -- minimum port number + - limit_max -- maximum port number + + Returns: + - a random port number + ''' + port = random.randint(limit_min, limit_max) + while is_used_port(uci, port): + port = random.randint(limit_min, limit_max) + return port + +def is_used_port(uci, port): + ''' + Check if a port is already used by another OpenVPN. + + Arguments: + - uci -- EUci instance + - port -- port to check + + Returns: + - True if the port is already used, False otherwise + ''' + for v in uci.get_all('openvpn'): + rport = int(uci.get('openvpn', v, 'port', default="0")) + lport = int(uci.get('openvpn', v, 'lport', default="0")) + if port == rport or port == lport: + return True + return False + +def list_connected_clients(openvpninstance, type="subnet"): + """ + List connected clients. + + Arguments: + - openvpninstance -- OpenVPN instance name + - type -- type of list (subnet or p2p) + + Returns: + - a list of connected clients + """ + try: + p = subprocess.run(["/usr/bin/openvpn-status", "-t", type, f"/var/run/openvpn_{openvpninstance}.socket"], check=True, capture_output=True, text=True) + except: + return {} + return json.loads(p.stdout) diff --git a/packages/python3-nethsec/src/nethsec/reverse_proxy/__init__.py b/packages/python3-nethsec/src/nethsec/reverse_proxy/__init__.py new file mode 100644 index 000000000..3e239b1de --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/reverse_proxy/__init__.py @@ -0,0 +1,249 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Reverse proxy helper functions + +All the functions in this module are meant to be used by the reverse proxy api and migration script. +Be careful when using them in other contexts. +""" + +import ipaddress +import os +import subprocess + +from euci import EUci + +from nethsec import utils + + +def set_proxy_pass(e_uci, location, proxy_pass): + """ + Set the proxy pass for the specified location. + This function is a mere workaround for nginx to allow non-resolvable domains to be used as proxy_pass. + + Args: + - e_uci: EUci object + - location: nginx config location id + - proxy_pass: string of the proxy pass to set + """ + prefix = proxy_pass.split('://')[0] + destination = proxy_pass.split('://')[1] + address = destination.split('/')[0] + if len(destination.split('/')) > 1: + path = '/' + '/'.join(destination.split('/')[1:]) + else: + path = '' + + try: + ipaddress.ip_address(address) + e_uci.set('nginx', location, 'proxy_pass', proxy_pass) + except ValueError: + e_uci.set('nginx', location, 'resolver', '127.0.0.1') + e_uci.set('nginx', location, 'set', [f'$upstream {address}']) + e_uci.set('nginx', location, 'proxy_pass', f'{prefix}://$upstream{path}') + + +def create_location(e_uci, uci_server, location, proxy_pass, domain='', allow=None): + """ + Create a new location in the nginx config. + + Args: + - e_uci: EUci object + - uci_server: where the location is defined + - location: path where the location will answer + - proxy_pass: where the location will proxy to + - domain: optional domain to set in the Host header + - allow: array of allowed ip addresses + + Returns: + location id of the created location + """ + if allow is None: + allow = [] + location_id = utils.get_random_id() + e_uci.set('nginx', location_id, 'location') + # defaults + e_uci.set('nginx', location_id, 'proxy_http_version', '1.1') + default_headers = [ + 'X-Forwarded-For $proxy_add_x_forwarded_for', + 'X-Real-IP $remote_addr', + 'Upgrade $http_upgrade', + 'Connection "upgrade"' + ] + if domain: + default_headers.append(f'Host {domain}') + e_uci.set('nginx', location_id, 'proxy_set_header', default_headers) + # set body limit to 1GB: same value of NethServer 7 + e_uci.set('nginx', location_id, 'client_max_body_size', '1024m') + # setup location + e_uci.set('nginx', location_id, 'uci_server', uci_server) + e_uci.set('nginx', location_id, 'location', location) + if len(allow) > 0: + e_uci.set('nginx', location_id, 'allow', allow) + set_proxy_pass(e_uci, location_id, proxy_pass) + + return location_id + + +def add_path(path, destination, description, allow): + """ + Add a new location to the _lan server. + + Args: + - path: path where the location will answer + - destination: destination endpoint + - description: description of the location + - allow: array of allowed ip addresses + """ + e_uci = EUci() + location = create_location(e_uci, '_lan', path, destination, allow=allow) + e_uci.set('nginx', location, 'uci_description', description) + # defaults + e_uci.set('nginx', location, 'proxy_ssl_verify', 'off') + # add location to _lan server + includes = list(e_uci.get('nginx', '_lan', 'include', list=True)) + if 'conf.d/_lan[.]proxy' not in includes: + includes.append('conf.d/_lan[.]proxy') + e_uci.set('nginx', '_lan', 'include', includes) + + e_uci.save('nginx') + + +def add_domain(domain, destination, certificate, description, allow): + """ + Add a new server to the nginx config. + + Args: + - domain: domain to answer to + - destination: where to proxy the request to + - certificate: certificate to be used, beware the certificate must be in the list of certificates + - description: easy description of the server + - allow: ip addresses allowed to access the server + """ + e_uci = EUci() + # create server instance + server_name = utils.get_random_id() + e_uci.set('nginx', server_name, 'server') + # create default location + create_location(e_uci, server_name, '/', destination, domain, allow) + # defaults + e_uci.set('nginx', server_name, 'proxy_ssl_verify', 'off') + e_uci.set('nginx', server_name, 'ssl_session_timeout', '64m') + e_uci.set('nginx', server_name, 'ssl_session_cache', 'shared:SSL:32k') + e_uci.set('nginx', server_name, 'listen', ['443 ssl', '[::]:443 ssl']) + e_uci.set('nginx', server_name, 'include', [f'conf.d/{server_name}.proxy']) + e_uci.set('nginx', server_name, 'access_log', 'syslog:server=unix:/dev/log') + e_uci.set('nginx', server_name, 'error_log', 'syslog:server=unix:/dev/log') + # setup server + valid_certificates = {name: certificate for (name, certificate) in certificate_list(e_uci).items() + if 'cert_path' in certificate} + e_uci.set('nginx', server_name, 'server_name', domain) + e_uci.set('nginx', server_name, 'ssl_certificate', valid_certificates[certificate]['cert_path']) + e_uci.set('nginx', server_name, 'ssl_certificate_key', valid_certificates[certificate]['key_path']) + e_uci.set('nginx', server_name, 'uci_description', description) + + e_uci.save('nginx') + + +def certificate_list(uci: EUci): + """ + List all the certificates in the system. + + Args: + - uci: euci object + """ + default_certificate = uci.get('nginx', '_lan', 'ssl_certificate') + + servers = utils.get_all_by_type(uci, 'nginx', 'server') + certificate_domain = {} + for server in servers: + if 'ssl_certificate' in servers[server]: + if server not in certificate_domain: + certificate_domain[servers[server]['ssl_certificate']] = [] + certificate_domain[servers[server]['ssl_certificate']].append(servers[server]['server_name']) + + certificates = {} + + # scan default certificates + for entry in os.scandir('/etc/nginx/conf.d'): + if entry.is_file() and entry.name.endswith('.crt') and os.path.isfile(entry.path[:-4] + '.key'): + certificates[entry.name[:-4]] = { + 'type': 'self-signed', + 'cert_path': entry.path, + 'key_path': entry.path[:-4] + '.key', + 'default': default_certificate == entry.path, + } + + # scan custom certificates + if not os.path.isdir('/etc/nginx/custom_certs'): + os.mkdir('/etc/nginx/custom_certs') + for entry in os.scandir('/etc/nginx/custom_certs'): + if entry.is_file() and entry.name.endswith('.crt') and os.path.isfile(entry.path[:-4] + '.key'): + certificates[entry.name[:-4]] = { + 'type': 'custom', + 'cert_path': entry.path, + 'key_path': entry.path[:-4] + '.key', + 'default': default_certificate == entry.path, + } + + # scan acme certificates + requested_certificates = utils.get_all_by_type(uci, 'acme', 'cert') + enabled_certificates = [certificate for certificate in requested_certificates + if requested_certificates[certificate]['enabled'] == '1'] + for certificate in enabled_certificates: + domain = requested_certificates[certificate]['domains'][0] + cert_path = f'/etc/ssl/acme/{domain}.fullchain.crt' + key_path = f'/etc/ssl/acme/{domain}.key' + if os.path.isfile(cert_path): + certificates[certificate] = { + 'type': 'acme', + 'pending': False, + 'requested_domains': requested_certificates[certificate]['domains'], + 'cert_path': cert_path, + 'key_path': key_path, + 'default': default_certificate == cert_path + } + else: + certificates[certificate] = { + 'type': 'acme', + 'pending': True, + 'requested_domains': requested_certificates[certificate]['domains'], + } + + for certificate in certificates: + if 'cert_path' in certificates[certificate]: + # get certificate details + details = subprocess.run( + ['openssl', 'x509', '-noout', '-text', '-in', certificates[certificate]['cert_path']], + check=True, capture_output=True, text=True) + # get certificate expiration + expiration = subprocess.run(['openssl', 'x509', '-noout', '-enddate', '-dateopt', 'iso_8601', '-in', + certificates[certificate]['cert_path']], + check=True, capture_output=True, text=True) + # fetch subject common name + subject = subprocess.run(['openssl', 'x509', '-noout', '-subject', '-nameopt', 'multiline', '-in', + certificates[certificate]['cert_path']], + check=True, capture_output=True, text=True) + cn = '' + for row in subject.stdout.split('\n'): + if 'commonName' in row: + cn = row.split('=')[1].strip() + break + + # get certificate assigned domains + servers = [] + if certificates[certificate]['cert_path'] in certificate_domain: + servers = certificate_domain[certificates[certificate]['cert_path']] + + # store certificate details + certificates[certificate]['details'] = details.stdout + certificates[certificate]['expiration'] = expiration.stdout.split('=')[1].strip() + certificates[certificate]['servers'] = servers + certificates[certificate]['domain'] = cn + + return certificates diff --git a/packages/python3-nethsec/src/nethsec/snort/__init__.py b/packages/python3-nethsec/src/nethsec/snort/__init__.py new file mode 100644 index 000000000..dc03ea423 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/snort/__init__.py @@ -0,0 +1,466 @@ +# Copyright (c) 2011 Jason Ish +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +""" Module for parsing Snort-like rules. + +Parsing is done using regular expressions and the job of this module +is to do its best at parsing out fields of interest from the rule +rather than perform a sanity check. + +The methods that parse multiple rules for a provided input +(parse_file, parse_fileobj) return a list of rules instead of dict +keyed by ID as its not the job of this module to detect or deal with +duplicate signature IDs. +""" + +import sys +import re +import logging +import io + +logger = logging.getLogger(__name__) + +# Compile an re pattern for basic rule matching. +rule_pattern = re.compile(r"^(?P#)*[\s#]*" + r"(?P" + r"(?P
[^()]+)" + r"\((?P.*)\)" + r"$)") + +# Rule actions we expect to see. +actions = ( + "alert", "config", "log", "pass", "activate", "dynamic", "drop", "reject", "sdrop") + +class Rule(dict): + """Class representing a rule. + + The Rule class is a class that also acts like a dictionary. + + Dictionary fields: + + - **group**: The group the rule belongs to, typically the filename. + - **enabled**: True if rule is enabled (uncommented), False is + disabled (commented) + - **action**: The action of the rule (alert, pass, etc) as a + string + - **proto**: The protocol string of the rule. + - **source_addr**: The source address string of the rule. + - **source_port**: The source ports string of the rule. + - **direction**: The direction string of the rule. + - **dest_addr**: The destination address string of the rule. + - **dest_port**: The destination ports string of the rule. + - **gid**: The gid of the rule as an integer + - **sid**: The sid of the rule as an integer + - **rev**: The revision of the rule as an integer + - **msg**: The rule message as a string + - **flowbits**: List of flowbit options in the rule + - **metadata**: Metadata values as a list + - **references**: References as a list + - **classtype**: The classification type + - **priority**: The rule priority, 0 if not provided + - **raw**: The raw rule as read from the file or buffer + + :param enabled: Optional parameter to set the enabled state of the rule + :param action: Optional parameter to set the action of the rule + :param group: Optional parameter to set the group (filename) of the rule + + """ + def __init__(self, enabled=None, action=None, group=None): + dict.__init__(self) + self["enabled"] = enabled + self["action"] = action + self["proto"] = None + self["source_addr"] = None + self["source_port"] = None + self["direction"] = None + self["dest_addr"] = None + self["dest_port"] = None + self["group"] = group + self["gid"] = 1 + self["sid"] = None + self["rev"] = None + self["msg"] = None + self["flowbits"] = [] + self["metadata"] = [] + self["references"] = [] + self["classtype"] = None + self["priority"] = 0 + + self["options"] = [] + + self["raw"] = None + + def __getattr__(self, name): + return self[name] + + @property + def id(self): + """ The ID of the rule. + + :returns: A tuple (gid, sid) representing the ID of the rule + :rtype: A tuple of 2 ints + """ + return (int(self.gid), int(self.sid)) + + @property + def idstr(self): + """Return the gid and sid of the rule as a string formatted like: + '[GID:SID]'""" + return "[%s:%s]" % (str(self.gid), str(self.sid)) + + def brief(self): + """ A brief description of the rule. + + :returns: A brief description of the rule + :rtype: string + """ + return "%s[%d:%d] %s" % ( + "" if self.enabled else "# ", self.gid, self.sid, self.msg) + + def __hash__(self): + return self["raw"].__hash__() + + def __str__(self): + """ The string representation of the rule. + + If the rule is disabled it will be returned as commented out. + """ + return self.format() + + def format(self): + return u"%s%s" % (u"" if self.enabled else u"# ", self.raw) + + def rebuild_options(self): + """ Rebuild the rule options from the list of options.""" + options = [] + for option in self.options: + if option["value"] is None: + options.append(option["name"]) + else: + options.append("%s:%s" % (option["name"], option["value"])) + return "%s;" % "; ".join(options) + +def remove_option(rule, name): + rule["options"] = [ + option for option in rule["options"] if option["name"] != name] + new_rule_string = "%s%s (%s)" % ( + "" if rule.enabled else "# ", + rule["header"].strip(), + rule.rebuild_options()); + return parse(new_rule_string, rule["group"]) + +def add_option(rule, name, value, index=None): + option = { + "name": name, + "value": value, + } + if index is None: + rule["options"].append(option) + else: + rule["options"].insert(index, option) + new_rule_string = "%s%s (%s)" % ( + "" if rule.enabled else "# ", + rule["header"].strip(), + rule.rebuild_options()) + return parse(new_rule_string, rule["group"]) + +def find_opt_end(options): + """ Find the end of an option (;) handling escapes. """ + offset = 0 + + while True: + i = options[offset:].find(";") + if options[offset + i - 1] == "\\": + offset += 2 + else: + return offset + i + +def parse(buf, group=None): + """ Parse a single rule for a string buffer. + + :param buf: A string buffer containing a single Snort-like rule + + :returns: An instance of of :py:class:`.Rule` representing the parsed rule + """ + + if type(buf) == type(b""): + buf = buf.decode("utf-8") + buf = buf.strip() + + m = rule_pattern.match(buf) + if not m: + return None + + if m.group("enabled") == "#": + enabled = False + else: + enabled = True + + header = m.group("header").strip() + + # If a decoder rule, the header will be one word. + if len(header.split(" ")) == 1: + action = header + proto = None + source_addr = None + source_port = None + direction = None + dest_addr = None + dest_port = None + else: + states = ["action", + "proto", + "source_addr", + "source_port", + "direction", + "dest_addr", + "dest_port", + ] + state = 0 + + rem = header + while state < len(states): + if not rem: + return None + if rem[0] == "[": + end = rem.find("]") + if end < 0: + return + end += 1 + token = rem[:end].strip() + rem = rem[end:].strip() + else: + end = rem.find(" ") + if end < 0: + token = rem + rem = "" + else: + token = rem[:end].strip() + rem = rem[end:].strip() + + if states[state] == "action": + action = token + elif states[state] == "proto": + proto = token + elif states[state] == "source_addr": + source_addr = token + elif states[state] == "source_port": + source_port = token + elif states[state] == "direction": + direction = token + elif states[state] == "dest_addr": + dest_addr = token + elif states[state] == "dest_port": + dest_port = token + + state += 1 + + if action not in actions: + return None + + rule = Rule(enabled=enabled, action=action, group=group) + rule["header"] = header + rule["proto"] = proto + rule["source_addr"] = source_addr + rule["source_port"] = source_port + rule["direction"] = direction + rule["dest_addr"] = dest_addr + rule["dest_port"] = dest_port + + options = m.group("options") + + while True: + if not options: + break + index = find_opt_end(options) + if index < 0: + raise Exception("end of option not found: %s" % (buf)) + option = options[:index].strip() + options = options[index + 1:].strip() + + if option.find(":") > -1: + name, val = [x.strip() for x in option.split(":", 1)] + else: + name = option + val = None + + rule["options"].append({ + "name": name, + "value": val, + }) + + if name in ["gid", "sid", "rev"]: + rule[name] = int(val) + elif name == "metadata": + if not name in rule: + rule[name] = [] + rule[name] += [v.strip() for v in val.split(",")] + elif name == "flowbits": + rule.flowbits.append(val) + elif name == "reference": + rule.references.append(val) + elif name == "msg": + if val.startswith('"') and val.endswith('"'): + val = val[1:-1] + rule[name] = val + else: + rule[name] = val + + if rule["msg"] is None: + rule["msg"] = "" + + rule["raw"] = m.group("raw").strip() + + return rule + +def parse_fileobj(fileobj, group=None): + """ Parse multiple rules from a file like object. + + Note: At this time rules must exist on one line. + + :param fileobj: A file like object to parse rules from. + + :returns: A list of :py:class:`.Rule` instances, one for each rule parsed + """ + rules = [] + buf = "" + for line in fileobj: + try: + if type(line) == type(b""): + line = line.decode() + except: + pass + if line.rstrip().endswith("\\"): + buf = "%s%s " % (buf, line.rstrip()[0:-1]) + continue + try: + rule = parse(buf + line, group) + if rule: + rules.append(rule) + except: + logger.error("failed to parse rule: %s" % (buf)) + raise + buf = "" + return rules + +def parse_file(filename, group=None): + """ Parse multiple rules from the provided filename. + + :param filename: Name of file to parse rules from + + :returns: A list of :py:class:`.Rule` instances, one for each rule parsed + """ + with io.open(filename, encoding="utf-8") as fileobj: + return parse_fileobj(fileobj, group) + +class FlowbitResolver(object): + + setters = ["set", "setx", "unset", "toggle"] + getters = ["isset", "isnotset"] + + def __init__(self): + self.enabled = [] + + def resolve(self, rules): + required = self.get_required_flowbits(rules) + enabled = self.set_required_flowbits(rules, required) + if enabled: + self.enabled += enabled + return self.resolve(rules) + return self.enabled + + def set_required_flowbits(self, rules, required): + enabled = [] + for rule in [rule for rule in rules.values() if not rule.enabled]: + for option, value in map(self.parse_flowbit, rule.flowbits): + if option in self.setters and value in required: + rule.enabled = True + enabled.append(rule) + return enabled + + def get_required_rules(self, rulemap, flowbits, include_enabled=False): + """Returns a list of rules that need to be enabled in order to satisfy + the list of required flowbits. + + """ + required = [] + + for rule in [rule for rule in rulemap.values()]: + if not rule: + continue + for option, value in map(self.parse_flowbit, rule.flowbits): + if option in self.setters and value in flowbits: + if rule.enabled and not include_enabled: + continue + required.append(rule) + + return required + + def get_required_flowbits(self, rules): + required_flowbits = set() + for rule in [rule for rule in rules.values() if rule and rule.enabled]: + for option, value in map(self.parse_flowbit, rule.flowbits): + if option in self.getters: + required_flowbits.add(value) + return required_flowbits + + def parse_flowbit(self, flowbit): + tokens = flowbit.split(",", 1) + if len(tokens) == 1: + return tokens[0], None + elif len(tokens) == 2: + return tokens[0], tokens[1] + else: + raise Exception("Flowbit parse error on %s" % (flowbit)) + +def enable_flowbit_dependencies(rulemap): + """Helper function to resolve flowbits, wrapping the FlowbitResolver + class. """ + resolver = FlowbitResolver() + return resolver.resolve(rulemap) + +def format_sidmsgmap(rule): + """ Format a rule as a sid-msg.map entry. """ + try: + return " || ".join([str(rule.sid), rule.msg] + rule.references) + except: + logger.error("Failed to format rule as sid-msg.map: %s" % (str(rule))) + return None + +def format_sidmsgmap_v2(rule): + """ Format a rule as a v2 sid-msg.map entry. + + eg: + gid || sid || rev || classification || priority || msg || ref0 || refN + """ + try: + return " || ".join([ + str(rule.gid), str(rule.sid), str(rule.rev), + "NOCLASS" if rule.classtype is None else rule.classtype, + str(rule.priority), rule.msg] + rule.references) + except: + logger.error("Failed to format rule as sid-msg-v2.map: %s" % ( + str(rule))) + return None diff --git a/packages/python3-nethsec/src/nethsec/users/__init__.py b/packages/python3-nethsec/src/nethsec/users/__init__.py new file mode 100644 index 000000000..429ae3d36 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/users/__init__.py @@ -0,0 +1,932 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Library that handles users and groups +""" + +import base64 +import hashlib +import ipaddress +import json +import os +import subprocess +import secrets + +from euci import EUci +from uci import UciExceptionNotFound + +from nethsec import utils +from nethsec.ldif import LDIFParser +from urllib.parse import urlparse +from passlib import hash +from io import BytesIO + + +def get_database_type(uci, database): + ''' + Retrieve database type + + Arguments: + - uci -- EUci pointer + - database -- Database identifier + + Returns: + - Database type (local or ldap) + ''' + return uci.get('users', database, default='') + +def get_user_by_name(uci, name, database="main"): + ''' + Retrieve a user by name + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Local database identifier (default: main) + + Returns: + - A user object or None if not found + ''' + user = {"local": get_database_type(uci, database) == "local"} + for u in uci.get_all('users'): + if uci.get('users', u, 'database', default='') != database: + continue + if uci.get('users', u, default='') == "user": + if uci.get('users', u, 'name', default='').lower() == name.lower(): + for opt in uci.get_all("users", u): + user[opt] = uci.get_all('users', u, opt) + # convert tuple to list + if type(user[opt]) is tuple: + user[opt] = list(user[opt]) + if user["local"]: + user["admin"] = is_admin(uci, name) + else: + user["admin"] = False + user['id'] = u + return user + return None + +def get_group_by_name(uci, name, database="main"): + ''' + Retrieve a group by name + + Arguments: + - uci -- EUci pointer + - name -- Group name + - database -- Local database identifier (default: main) + + + Returns: + - A group object or None if not found + ''' + group = {"local": get_database_type(uci, database) == "local"} + for g in uci.get_all('users'): + if uci.get('users', g, 'database', default='') != database: + continue + if uci.get('users', g, default='') == "group": + if uci.get('users', g, 'name', default='') == name: + for opt in uci.get_all("users", g): + group[opt] = uci.get_all('users', g, opt) + # convert tuple to list + if type(group[opt]) is tuple: + group[opt] = list(group[opt]) + group['id'] = g + return group + return None + +def get_user_addresses(uci, user): + ''' + Retrieve all IP addresses associated to given user + + Arguments: + - uci -- EUci pointer + - user -- User object id (UCI section) + + Returns a tuple of lists: + - first element is a list of IPv4 addresses + - second element is a list of IPv6 addresses + ''' + ipv4 = [] + ipv6 = [] + user_obj = get_user_by_name(uci, user) + # get addresses from plain ipaddr option + for ip in uci.get('users', user_obj['id'], 'ipaddr', list=True, default=[]): + if ':' in ip: + ipv6.append(ip) + else: + ipv4.append(ip) + # get vpn reservation + ovpn_ipaddr = uci.get('users', user_obj['id'], 'openvpn_ipaddr', default="") + if ovpn_ipaddr: + if ':' in ovpn_ipaddr: + ipv6.append(ovpn_ipaddr) + else: + ipv4.append(ovpn_ipaddr) + # get address from DNS record and DHCP reservation + for st in ['host', 'domain']: + for h in uci.get('objects', user_obj['id'], st, list=True, default=[]): # dhcp reservation + ip = uci.get('dhcp', h, 'ip', default='') + if not ip: + continue + if ':' in ip: + ipv6.append(ip) + else: + ipv4.append(ip) + + return (ipv4, ipv6) + +def get_user_macs(uci, user): + ''' + Retrieve all MAC addresses associated to given user + + Arguments: + - uci -- EUci pointer + - user -- User object id (UCI section) + + Returns: + - A list of MAC addresses + ''' + user_obj = get_user_by_name(uci, user) + return list(uci.get('users', user_obj['id'], 'macaddr', list=True, default=[])) + +def get_group_addresses(uci, group): + ''' + Retrieve all IP addresses associated to given group + + Arguments: + - uci -- EUci pointer + - user -- Group object id (UCI section) + + Returns: + - A tuple of lists: + - first element is a list of IPv4 addresses + - second element is a list of IPv6 addresses + ''' + ipv4 = [] + ipv6 = [] + group_obj = get_group_by_name(uci, group) + for u in uci.get('users', group_obj['id'], 'user', list=True, default=[]): + (uipv4, uipv6) = get_user_addresses(uci, u) + ipv4 = ipv4 + uipv4 + ipv6 = ipv6 + uipv6 + return (ipv4, ipv6) + +def get_group_macs(uci, group): + ''' + Retrieve all MAC addresses associated to given group + + Arguments: + - uci -- EUci pointer + - group -- Group object id (UCI section) + + Returns: + - A list of MAC addresses + ''' + macs = [] + group_obj = get_group_by_name(uci, group) + for u in uci.get('users', group_obj['id'], 'user', list=True, default=[]): + macs = macs + get_user_macs(uci, u) + return macs + +def list_users(uci, database='main'): + ''' + Retrieve all users + + Arguments: + - database -- Database identifier (default: main) + + Returns: + - A list of user objects + ''' + users = [] + try: + dbconf = uci.get_all('users', database) + except: + raise utils.ValidationError('database', 'db_not_found', database) + dbtype = get_database_type(uci, database) + if dbtype == "local": + for u in utils.get_all_by_type(uci, 'users', 'user'): + if uci.get('users', u, 'database', default='') != database: + continue + if uci.get('users', u, default='') == "user": + username = uci.get('users', u, 'name', default='') + user = get_user_by_name(uci, username, database) + users.append(user) + users = sorted(users, key=lambda u: u['name']) + elif dbtype == "ldap": + # retrieve also user_cn attribute for old configurations + display_attr = dbconf.get('user_display_attr', dbconf.get('user_cn', 'cn')) + users = list_remote_users(dbconf.get('uri'), dbconf.get('user_dn'), dbconf.get('user_attr'), display_attr, dbconf.get('start_tls') == '1', dbconf.get('tls_reqcert'), dbconf.get('bind_dn'), dbconf.get('bind_password'), dbconf.get('schema')) + for u in users: + user = get_user_by_name(uci, u['name'], database) + if user: + u.update(user) + else: + u['id'] = None + u['local'] = False + u['admin'] = False + u['database'] = database + pass + + return users + +def add_ldap_database(uci, name, uri, schema, base_dn, user_dn, user_attr, user_display_attr, start_tls=False, tls_reqcert='never', description="", bind_dn=None, bind_password=None, user_bind_dn=None): + ''' + Add a new LDAP database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + - uri -- LDAP URI + - schema -- LDAP schema + - base_dn -- LDAP base DN + - user_dn -- LDAP user DN + - user_attr -- LDAP user attribute + - user_display_attr -- LDAP user full name attribute + - start_tls -- Use TLS (default: False) + - tls_reqcert -- TLS certificate validation (default: never) + - description -- Database description (default: "") + - bind_dn -- LDAP bind DN + - bind_password -- LDAP bind password + - user_bind_dn -- LDAP custom user bind DN + + Returns: + - The database identifier + ''' + if uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_already_exists', name) + ldap = uci.set('users', name, 'ldap') + uci.set('users', name, 'uri', uri) + uci.set('users', name, 'schema', schema) + uci.set('users', name, 'base_dn', base_dn) + uci.set('users', name, 'user_dn', user_dn) + uci.set('users', name, 'user_attr', user_attr) + uci.set('users', name, 'user_display_attr', user_display_attr) + uci.set('users', name, 'start_tls', '1' if start_tls else '0') + uci.set('users', name, 'tls_reqcert', tls_reqcert) + uci.set('users', name, 'description', description) + if bind_dn and bind_password: + uci.set('users', name, 'bind_dn', bind_dn) + uci.set('users', name, 'bind_password', bind_password) + if user_bind_dn: + uci.set('users', name, 'user_bind_dn', user_bind_dn) + uci.save("users") + return ldap + +def edit_ldap_database(uci, name, uri, schema, base_dn, user_dn, user_attr, user_display_attr, start_tls=False, tls_reqcert='never', description="", bind_dn=None, bind_password=None, user_bind_dn=None): + ''' + Edit an existing LDAP database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + - uri -- LDAP URI + - schema -- LDAP schema + - base_dn -- LDAP base DN + - user_dn -- LDAP user DN + - user_attr -- LDAP user attribute + - user_display_attr -- LDAP user full name attribute + - start_tls -- Use TLS (default: False) + - tls_reqcert -- TLS certificate validation (default: never) + - description -- Database description (default: "") + - bind_dn -- LDAP bind DN + - bind_password -- LDAP bind password + - user_bind_dn -- LDAP custom user bind DN + + Returns: + - The database identifier + ''' + if not uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_not_found', name) + if uci.get('users', name, default='') != "ldap": + raise utils.ValidationError('name', 'db_not_ldap', name) + uci.set('users', name, 'uri', uri) + uci.set('users', name, 'schema', schema) + uci.set('users', name, 'base_dn', base_dn) + uci.set('users', name, 'user_dn', user_dn) + uci.set('users', name, 'user_attr', user_attr) + uci.set('users', name, 'user_display_attr', user_display_attr) + uci.set('users', name, 'start_tls', '1' if start_tls else '0') + uci.set('users', name, 'tls_reqcert', tls_reqcert) + uci.set('users', name, 'description', description) + if bind_dn and bind_password: + uci.set('users', name, 'bind_dn', bind_dn) + uci.set('users', name, 'bind_password', bind_password) + else: + try: + uci.delete('users', name, 'bind_dn') + uci.delete('users', name, 'bind_password') + except: + pass + if user_bind_dn: + uci.set('users', name, 'user_bind_dn', user_bind_dn) + else: + try: + uci.delete('users', name, 'user_bind_dn') + except: + pass + # remove old unused user_cn field, if present + try: + uci.delete('users', name, 'user_cn') + except: + pass + uci.save("users") + return True + +def delete_ldap_database(uci, name): + ''' + Delete an existing LDAP database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + + Returns: + - True if successful + ''' + if not uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_not_found', name) + if uci.get('users', name, default='') != "ldap": + raise utils.ValidationError('name', 'db_not_ldap', name) + for u in uci.get_all('users'): + if uci.get('users', u, 'database', default='') == name: + uci.delete('users', u) + uci.delete('users', name) + uci.save("users") + return True + +def add_local_database(uci, name, description=""): + ''' + Add a new local database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + - description -- Database description (default: "") + + Returns: + - The database identifier + ''' + if uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_already_exists', name) + local = uci.set('users', name, 'local') + uci.set('users', name, 'description', description) + uci.save("users") + return True + +def edit_local_database(uci, name, description=""): + ''' + Edit an existing local database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + - description -- Database description (default: "") + + Returns: + - The database identifier + ''' + if not uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_not_found', name) + if uci.get('users', name, default='') != "local": + raise utils.ValidationError('name', 'db_not_local', name) + uci.set('users', name, 'description', description) + uci.save("users") + return True + +def delete_local_database(uci, name): + ''' + Delete an existing local database + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + + Returns: + - True if successful + ''' + if not uci.get('users', name, default=None): + raise utils.ValidationError('name', 'db_not_found', name) + if uci.get('users', name, default='') != "local": + raise utils.ValidationError('name', 'db_not_local', name) + uci.delete('users', name) + for u in uci.get_all('users'): + if uci.get('users', u, 'database', default='') == name: + uci.delete('users', u) + uci.save("users") + return True + +def list_databases(uci): + ''' + Retrieve all databases + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of database objects, each one containing: + - name: database identifier + - type: database type (local or ldap) + - description: database description + ''' + ret = [] + for db in uci.get_all('users'): + if uci.get('users', db, default='') == "local": + ret.append({"name": db, "type": "local", "description": uci.get('users', db, 'description', default='')}) + elif uci.get('users', db, default='') == "ldap": + ret.append({"name": db, "type": "ldap", "description": uci.get('users', db, 'description', default=''), + "schema": uci.get('users', db, 'schema', default=''), + "uri": uci.get('users', db, 'uri', default=''), + "used": used_by(uci, db)}) + return ret + +def get_database(uci, name): + ''' + Retrieve a database by name + + Arguments: + - uci -- EUci pointer + - name -- Database identifier + + Returns: + - A database object or None if not found + ''' + try: + db = uci.get_all('users', name) + except: + return None + db["name"] = name + db["type"] = get_database_type(uci, name) + if 'user_cn' in db: + # migrate from old to new attribute names + db['user_display_attr'] = db['user_cn'] + return db + +def add_local_user(uci, name, password="", description="", database="main", extra_fields={}): + ''' + Add a new local user + + Arguments: + - uci -- EUci pointer + - name -- User name + - password -- User password + - description -- User description (default: "") + - database -- Local database identifier (default: main) + - extra_fields -- Extra fields to add to the user (default: {}) + + Returns: + - The user identifier + ''' + if get_user_by_name(uci, name, database): + raise utils.ValidationError('name', 'user_already_exists', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + id = utils.get_random_id() + user = uci.set('users', id, 'user') + uci.set('users', id, 'database', database) + uci.set('users', id, 'name', name) + if password: + uci.set('users', id, 'password', shadow_password(password)) + uci.set('users', id, 'description', description) + for key in extra_fields: + uci.set('users', id, key, extra_fields[key]) + uci.save("users") + return id + +def edit_local_user(uci, name, password="", description=None, database="main", extra_fields={}): + ''' + Edit an existing local user + + Arguments: + - uci -- EUci pointer + - name -- User name + - password -- User password + - description -- User description (default: None) + - database -- Local database identifier (default: main) + - extra_fields -- Extra fields to add to the user (default: {}) + + Returns: + - The user identifier + ''' + user = get_user_by_name(uci, name, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + if password: + shadow = shadow_password(password) + uci.set('users', user["id"], 'password', shadow) + # update password inside the rpcd configuration database + if is_admin(uci, name): + for l in utils.get_all_by_type(uci, 'rpcd', 'login'): + if uci.get('rpcd', l, 'username', default='') == name: + uci.set('rpcd', l, 'password', shadow) + uci.save("rpcd") + if description is not None: + uci.set('users', user["id"], 'description', description) + for key in uci.get_all('users', user["id"]): + if not key in ["name", "description", "password", "database"]: + uci.delete('users', user["id"], key) + for key in extra_fields: + uci.set('users', user["id"], key, extra_fields[key]) + uci.save("users") + return user["id"] + +def delete_local_user(uci, name, database="main"): + ''' + Delete an existing local user + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Local database identifier (default: main) + + Returns: + - True if successful + ''' + user = get_user_by_name(uci, name, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + # remove user from all groups + for g in uci.get_all('users'): + if uci.get('users', g, 'database', default='') != database: + continue + if uci.get('users', g, default='') == "group": + gusers = list(uci.get('users', g, 'user', list=True, default=[])) + if name in gusers: + gusers.remove(name) + uci.set('users', g, 'user', gusers) # the user field may be deleted by uci if the list is empty + uci.delete('users', user["id"]) + uci.save("users") + if is_admin(uci, name): + remove_admin(uci, name) + return True + +def add_local_group(uci, name, users=[], description="", database="main"): + ''' + Add a new local group + + Arguments: + - uci -- EUci pointer + - name -- Group name + - users -- List of users (default: []) + - description -- Group description (default: "") + - database -- Local database identifier (default: main) + + Returns: + - The group identifier + ''' + if get_group_by_name(uci, name, database): + raise utils.ValidationError('name', 'group_already_exists', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + + id = utils.get_random_id() + group = uci.set('users', id, 'group') + uci.set('users', id, 'database', database) + uci.set('users', id, 'name', name) + uci.set('users', id, 'description', description) + uci.set('users', id, 'user', users) + uci.save("users") + return id + +def edit_local_group(uci, name, users=[], description="", database="main"): + ''' + Edit an existing local group + + Arguments: + - uci -- EUci pointer + - name -- Group name + - users -- List of users (default: []) + - description -- Group description (default: "") + - database -- Local database identifier (default: main) + + Returns: + - The group identifier + ''' + group = get_group_by_name(uci, name, database) + if not group: + raise utils.ValidationError('name', 'group_not_found', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + uci.set('users', group["id"], 'user', users) + uci.set('users', group["id"], 'description', description) + uci.save("users") + return group["id"] + +def delete_local_group(uci, name, database="main"): + ''' + Delete an existing local group + + Arguments: + - uci -- EUci pointer + - name -- Group name + - database -- Local database identifier (default: main) + + Returns: + - True if successful + ''' + group = get_group_by_name(uci, name, database) + if not group: + raise utils.ValidationError('name', 'group_not_found', name) + if get_database_type(uci, database) != "local": + raise utils.ValidationError('database', 'db_not_local', database) + uci.delete('users', group["id"]) + uci.save("users") + return True + +def add_remote_user(uci, name, database, extra_fields={}): + ''' + Add a new remote user + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Database identifier + - extra_fields -- Extra fields to add to the user (default: {}) + + Returns: + - The user identifier + ''' + if get_user_by_name(uci, name, database): + raise utils.ValidationError('name', 'user_already_exists', name) + if get_database_type(uci, database) != "ldap": + + raise utils.ValidationError('database', 'db_not_ldap', database) + id = utils.get_random_id() + user = uci.set('users', id, 'user') + uci.set('users', id, 'database', database) + uci.set('users', id, 'name', name) + for key in extra_fields: + uci.set('users', id, key, extra_fields[key]) + uci.save("users") + return id + +def edit_remote_user(uci, name, database, extra_fields={}): + ''' + Edit an existing remote user + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Database identifier + - extra_fields -- Extra fields to add to the user (default: {}) + + Returns: + - The user identifier + ''' + user = get_user_by_name(uci, name, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', name) + if get_database_type(uci, database) != "ldap": + raise utils.ValidationError('database', 'db_not_ldap', database) + for key in uci.get_all('users', user["id"]): + if not key in ["name", "database"]: + uci.delete('users', user["id"], key) + for key in extra_fields: + uci.set('users', user["id"], key, extra_fields[key]) + uci.save("users") + return user["id"] + +def delete_remote_user(uci, name, database): + ''' + Delete an existing remote user + + Arguments: + - uci -- EUci pointer + - name -- User name + - database -- Database identifier + + Returns: + - True if successful + ''' + user = get_user_by_name(uci, name, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', name) + if get_database_type(uci, database) != "ldap": + raise utils.ValidationError('database', 'db_not_ldap', database) + uci.delete('users', user["id"]) + uci.save("users") + return True + +def shadow_password(password): + ''' + Generate a shadow password + + Arguments: + - password -- Clear text password + + Returns: + - A shadow password in crypt(3) format, as generate by mkpasswd. Format: $6$salt$hash + ''' + return hash.sha512_crypt.using(salt=secrets.token_hex(8), rounds=5000).hash(password) + +def check_password(password, shadow): + ''' + Check a shadow password + + Arguments: + - password -- Clear text password + - shadow -- Shadow password in crypt(3) format + + Returns: + - True if password matches, False otherwise + ''' + return hash.sha512_crypt.verify(password, shadow) + + +def check_old_password(username, password): + """ + Checks if the old password is correct, discriminates between root and other users. + + Arguments: + - username -- Username of the user to check + - password -- Password to check + + Returns: + - True if the password is correct, False otherwise + """ + data = { + "username": username, + "password": password, + "timeout": 1 + } + process = subprocess.run(["/bin/ubus", "call", "session", "login", json.dumps(data)], capture_output=True) + return process.returncode == 0 + + +def ldif2users(ldif_data, user_attr="uid", display_attr="cn"): + ''' + Parse an LDIF file and return a list of users + + Arguments: + - ldif_data -- LDIF data + - user_attr -- User attribute (default: uid) + - display_attr -- Display name attr (default: cn) + + Returns: + - A list of users + ''' + users = [] + user = None + ldif_data_bytes = ldif_data.encode('utf-8') + ldif_data_io = BytesIO(ldif_data_bytes) + + parser = LDIFParser(ldif_data_io) + for dn, record in parser.parse(): + if user_attr in record: + user = {} + user["name"] = record[user_attr][0].lower() + if display_attr in record: + user["description"] = record[display_attr][0] + else: + user["description"] = "" + users.append(user) + return users + +def list_remote_users(uri, user_dn, user_attr, user_display_attr, start_tls=False, tls_reqcert="never", bind_dn=None, bind_password=None, schema='ldap'): + ''' + Test LDAP connection + + Arguments: + - uri -- LDAP URI + - user_dn -- LDAP user DN + - user_attr -- LDAP user attribute + - user_display_attr -- LDAP user full name attribute + - start_tls -- Use TLS (default: False) + - tls_reqcert -- TLS certificate validation (default: never) + - bind_dn -- LDAP bind DN + - bind_password -- LDAP bind password + - schema -- LDAP schema, 'ad' or 'ldap' + + Returns: + - A list of users, each one containing: + - name: user name + - description: user description + ''' + env = os.environ.copy() + env['LDAPTLS_REQCERT'] = tls_reqcert + omatch = '' + try: + # -E option executes a paged search without user prompt + # -LLL option suppresses LDAP version and search result headers + cmd = ["ldapsearch", "-LLL", "-x", "-H", uri, "-E", "pr=1000/noprompt", "-b", user_dn] + if start_tls: + cmd.append("-ZZ") + if bind_dn and bind_password: + cmd.extend(["-D", bind_dn, "-w", bind_password]) + if schema == "ad": + omatch = "(objectClass=person)" + else: + omatch = "(objectClass=posixAccount)" + cmd.extend(["-S", user_attr]) # request sorting + cmd.extend([omatch, user_display_attr, user_attr, user_display_attr]) + p = subprocess.run(cmd, env=env, capture_output=True, text=True) + return ldif2users(p.stdout, user_attr, user_display_attr) + except subprocess.CalledProcessError as e: + return [] + +def set_admin(uci, username, database): + ''' + Set a user as admin by creating a login record in rpcd configuration database + + Arguments: + - uci -- EUci pointer + - username -- User name + - database -- Database identifier + + Returns: + - The user identifier inside the rpcd configuration database + ''' + user = get_user_by_name(uci, username, database) + if not user: + raise utils.ValidationError('name', 'user_not_found', username) + logins = utils.get_all_by_type(uci, 'rpcd', 'login') + for l in logins: + if logins[l].get("username") == username: + raise utils.ValidationError('name', 'admin_user_already_exists', username) + id = utils.get_random_id() + uci.set('rpcd', id, 'login') + uci.set('rpcd', id, 'username', username) + uci.set('rpcd', id, 'password', user["password"]) + uci.set('rpcd', id, 'read', '*') + uci.set('rpcd', id, 'write', '*') + uci.save("rpcd") + return id + +def remove_admin(uci, username): + ''' + Remove a user from rpcd configuration database + + Arguments: + - uci -- EUci pointer + - username -- User name + + Returns: + - True if successful + ''' + logins = utils.get_all_by_type(uci, 'rpcd', 'login') + for l in logins: + if logins[l].get("username") == username: + uci.delete('rpcd', l) + uci.save("rpcd") + return True + raise utils.ValidationError('name', 'admin_user_not_found', username) + +def is_admin(uci, username): + ''' + Check if a user is admin + + Arguments: + - uci -- EUci pointer + - username -- User name + + Returns: + - True if user is admin, False otherwise + ''' + logins = utils.get_all_by_type(uci, 'rpcd', 'login') + if logins is None: + return False + for l in logins: + if logins[l].get("username") == username: + return True + return False + + +def used_by(uci, database_name): + """ + Checks if the database is used by VPN or other services + + Arguments: + - uci -- EUci pointer + - database_name -- Database identifier + + Returns: + - dict containing the service that the database is used by + """ + results = [] + try: + for instance in uci.get_all('openvpn'): + if uci.get('openvpn', instance, 'ns_user_db', default='') == database_name: + results.append('openvpn') + break + except UciExceptionNotFound: + pass + + try: + for instance in uci.get_all('network'): + # could filter by proto = 'wireguard' but the performance is not an issue + if uci.get('network', instance, 'ns_user_db', default='') == database_name: + results.append('wireguard') + break + except UciExceptionNotFound: + pass + + return results diff --git a/packages/python3-nethsec/src/nethsec/utils/__init__.py b/packages/python3-nethsec/src/nethsec/utils/__init__.py new file mode 100644 index 000000000..fc3a85801 --- /dev/null +++ b/packages/python3-nethsec/src/nethsec/utils/__init__.py @@ -0,0 +1,555 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +General utilities +''' + +import os +import base64 +import re +import json +import uuid +import hashlib +import subprocess + +def get_random_id(): + ''' + Return a random valid UCI id. + + Random ids: + - have a length of 11 characters + - are sanitized accordingly to UCI conventions (see 'sanitize' function) + - start with ns\_ prefix + + Arguments: + - name -- the name of the section + + Returns: + - a valid UCI identifier as string + ''' + + h = hashlib.new('sha1') + h.update(uuid.uuid4().bytes) + digest = h.hexdigest() + return get_id(digest, 11) + +def get_id(name, length = 100): + ''' + Return a valid UCI id based on the given string. + All auto-generated NethSecurity ids: + + - have a maximum length of 100 characters + - start with ns\_ prefix + - are sanitized accordingly to UCI conventions + + Arguments: + - name -- the name of the section + - length -- maximum id length, default is 100. Maximum length for firewall zones is 15. + + Returns: + - a valid UCI identifier as string + ''' + sname = f'ns_{sanitize(name)}' + return sname[0:length] + +def sanitize(name): + ''' + Replace illegal chars with _ char. + UCI identifiers and config file names may contain only the characters a-z, 0-9 and _ + + Arguments: + - name -- the name of the section + + Returns: + - a string with valid chachars for UCI + ''' + name = re.sub(r'[^\x00-\x7F]+','_', name) + name = re.sub('[^0-9a-zA-Z]', '_', name) + name = name.removesuffix('_') + return name + +def get_all_by_type(uci, config, utype): + ''' + Return all sections of the given utype from the given config + + Arguments: + - uci -- EUci pointer + - config -- Configuration database name + - utype -- Section type + + Returns: + - A dictionary of all matched sections, None in case of error + ''' + ret = dict() + try: + for section in uci.get(config): + if uci.get(config, section) == utype: + ret[section] = uci.get_all(config, section) + return ret + except: + return None + +def get_device_name(hwaddr): + ''' + Retrieve the physical device name given the MAC address + + Aarguments: + - hwaddr -- MAC address string + + Returns: + - The device name as a string if the network interface has been found, None otherwise. + ''' + try: + interfaces = json.loads(subprocess.run(["/sbin/ip", "--json", "address", "show"], check=True, capture_output=True).stdout) + for interface in interfaces: + if interface["address"] == hwaddr: + return interface["ifname"] + except: + return None + + return None + +def get_interface_from_mac(uci, hwaddr): + ''' + Retrieve the logical UCI interface name given the MAC address + + Arguments: + - uci -- EUci pointer + - hwaddr -- MAC address string + + Returns: + - The device name as a string if the interface has been found, None otherwise + ''' + device = get_device_name(hwaddr) + return get_interface_from_device(uci, device) + +def get_interface_from_device(uci, device): + ''' + Retrieve the logical UCI interface name given the device name + + Arguments: + - uci -- EUci pointer + - device -- Device name + + Returns: + - The device name as a string if the interface has been found, None otherwise + ''' + for section in uci.get("network"): + if uci.get("network", section) == "interface": + try: + proto = uci.get("network", section, 'proto') + except: + continue + if proto == 'bonding' and section == device: + return section + if proto == 'pppoe' and section == device.removeprefix('pppoe-'): + return section + try: + sdevice = uci.get("network", section, 'device') + except: + continue + if proto != 'bonding' and device == sdevice: + return section + + return None + +def get_all_by_option(uci, config, option, value, deep = True): + ''' + Return all sections with the given option value + + Arguments: + - uci -- EUci pointer + - config -- Configuration database name + - option -- Option name + - value -- Option value + - deep - If true, return a dict of all matched keys, otherwise return a list of section names + + Returns: + - A dictionary or a list of all matched sections + ''' + ret = dict() + for section in uci.get(config, list=True, default=[]): + if uci.get(config, section, option, default='') == value: + if deep: + ret[section] = uci.get_all(config, section) + else: + ret[section] = 1 + if deep: + return ret + else: + return list(ret.keys()) + + +def get_all_devices_by_zone(uci, zone, exclude_aliases=False): + ''' + Retrieve all devices associated to the given zone + + Arguments: + - uci -- EUci pointer + - zone -- Firewall zone name + + Returns: + - A list of device names + ''' + devices = [] + for section in uci.get("firewall"): + if uci.get("firewall", section, default='') == 'zone' and uci.get("firewall", section, "name", default='') == zone: + devices = devices + list(uci.get("firewall", section, "device", list=True, default=[])) + networks = uci.get("firewall", section, "network", list=True, default=[]) + for network in networks: + if uci.get('network', network, 'proto', default='') == 'wireguard': + devices.append(network) + continue + device = uci.get("network", network, "device", default="") + if exclude_aliases and device.startswith("@"): + continue + if device != "": + devices.append(device) + else: + name = uci.get("network", network, "name", default="") + if name != "": + devices.append(name) + + # remove duplicates + return list(set(devices)) + + +def get_all_wan_devices(uci, exclude_aliases=False): + """ + Retrieve all devices associated to the wan zone + + Arguments: + - uci -- EUci pointer + - exclude_aliases -- If true, exclude devices starting with @ + + Returns: + - A list of device names + """ + return get_all_devices_by_zone(uci, 'wan', exclude_aliases) + + +def get_all_lan_devices(uci): + ''' + Retrieve all devices associated to the lan zone, except for VPN ones + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of device names + ''' + # exclude tun and ipsec devices + return list(filter(lambda d: not d.startswith("ipsec") and not d.startswith("tun"), get_all_devices_by_zone(uci, 'lan'))) + +def get_unassigned_devices(uci): + ''' + Retrieve all unused/unassigned devices. + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of devices + ''' + unassigned = [] + try: + p = subprocess.run(["ip", "-j", "link"], check=True, capture_output=True, text=True) + devices = json.loads(p.stdout) + except: + return [] + + u_interfaces = get_all_by_type(uci, 'network', 'interface') + u_devices = get_all_by_type(uci, 'network', 'device') + + for ip_device in devices: + free = True + ifname = ip_device.get('ifname', '') + # exclude special devices + if not ifname or ifname == "lo" or ifname.startswith("ifb-"): + continue + # exclude tun devices and ppp devices + if ip_device.get("link_type", "ether") == "none" or ip_device.get("link_type", "ether") == "ppp": + continue + # skip bridged interfaces + if not ip_device.get("master") is None: + continue + # search among UCI devices + for d in u_devices: + # ports are present on bridge devices + try: + ports = uci.get_all('network', d, 'ports') + except: + ports = [] + if uci.get('network', d, 'name', default="") == ifname or ifname in ports: + free = False + # search among UCI interfaces + for i in u_interfaces: + # slaves are present on bond devices + slaves = [] + if uci.get('network', i, 'proto', default='') == 'bonding': + slaves = list(uci.get_all('network', i, 'slaves')) + # for bonds, the section name is the nmame of device with 'bond-' prefix + slaves.append(f'bond-{i}') + device = uci.get('network', i, 'device', default="") + if device == ifname or ifname in slaves: + free = False + # search inside hotspot configuration + if uci.get('dedalo', 'config', 'disabled', default="1") == "0" and uci.get('dedalo', 'config', 'interface', default="") == ifname: + free = False + if free: + unassigned.append(ifname) + + # prepare list of devices used in bridges + used = {} + for d in u_devices: + try: + for p in uci.get_all('network', d, 'ports'): + used[p] = 1 + except: + pass + # prepare list of devices used in bonds + bonds = get_all_by_type(uci, 'network', 'interface') + for b in bonds: + try: + for s in uci.get_all('network', b, 'slaves'): + used[s] = 1 + except: + pass + + used = list(used.keys()) + + for d in u_devices: + free = True + d_name = uci.get('network', d, 'name', default=None) + # skip a device which is already up and visibile inside ip command + # check if the device is used inside and interface + if get_interface_from_device(uci, d_name): + continue + # check if the device is used inside a zone + for z in get_all_by_type(uci, 'firewall', 'zone'): + try: + z_devices = uci.get_all('firewall', z, 'device') + except: + z_devices = [] + try: + z_networks = uci.get_all('firewall', z, 'network') + except: + z_networks = [] + if d_name in z_devices or d_name in z_networks or d_name in used: + free = False + continue + if free: + unassigned.append(d_name) + return unassigned + + +def validation_errors(errors): + ''' + Generate a validation error for the APIs from an array. + + Arguments: + - errors -- An array of array errors + + Each array element is an array composed by 3 parameters: + - the name of the parameter + - the validation error reason + - the parameter original value that caused the error + + Returns: + - A validation error object + ''' + verrors = [] + for e in errors: + verrors.append({"parameter": e[0], "message": e[1].strip().replace(" ", "_").lower(), "value": e[2]}) + + return {"validation": {"errors": verrors}} + +def validation_error(parameter, message="", value=""): + ''' + Generate a validation error for the APIs. + + Arguments: + - parameter -- The name of the parameter + - message -- The validation error reason, default is empty + - value -- The parmeter original value that caused the error + + Returns: + - A validation error object + ''' + return validation_errors([[parameter, message, value]]) + +def generic_error(error): + ''' + Generate a generic error for the APIs. + + Arguments: + - message -- An error message + Returns: + - A validation error object + ''' + return {"error": error.strip().replace(" ", "_").lower()} + +def shadow_password(password): + ''' + Generates a shadow password hash using SHA-512 algorithm. + + Arguments: + - password (str) - the password to be hashed. + + Returns: + - the shadow password hash as strin + ''' + salt = base64.b64encode(os.urandom(12)) + phash = base64.b64encode(hashlib.pbkdf2_hmac('sha512', bytes(password, 'UTF-8'), salt, 200000)) + return f"$6${salt.decode('UTF-8')}${phash.decode('UTF-8')}" + +def check_password(password, shadow): + ''' + Check if the given password matches the given shadow password hash. + + Arguments: + - password (str) - the password to be checked. + - shadow (str) - the shadow password hash to be checked. + + Returns: + - True if the password matches the hash, False otherwise + ''' + (_, alg, salt, curhash) = shadow.split("$") + phash = base64.b64encode(hashlib.pbkdf2_hmac('sha512', bytes(password, 'UTF-8'), salt.encode("UTF-8"), 200000)) + return phash.decode("UTF-8") == curhash + +def get_user_by_username(uci, username): + ''' + Retrieve the user object id (UCI section) given the username + + Arguments: + - uci -- EUci pointer + - username -- Username + + Returns: + - The user object id (UCI section) if the user has been found, None otherwise + ''' + users = get_all_by_type(uci, 'objects', 'user') + for user in get_all_by_type(uci, "users", "user"): + if users[user].get("username", "") == username: + users[user]["id"] = user + return users[user] + return None + +def get_all_device_ips(): + ''' + Retrieve all device IPs + ''' + ret = {} + p = subprocess.run(["/sbin/ip", "-j", "address"], check=True, text=True, capture_output=True) + data = json.loads(p.stdout) + for interface in data: + ipaddr = [] + name = interface.get("ifname") + for addr in interface.get("addr_info", []): + if name not in ret: + ret[name] = [] + ret[name].append(addr.get('local', '')) + return ret + + +def get_all_wan_ips(uci): + ''' + List all WAN interfaces with their IP addresses + Exclude bond management IPs. + + Arguments: + - uci -- EUci pointer + + Returns: + - A list of dictionaries with device and IP address + ''' + ret = [] + seen = set() # track seen devices to avoid duplicates + ips = get_all_device_ips() + wans = get_all_wan_devices(uci) + # list configured static wans + for wan in wans: + interface = get_interface_from_device(uci, wan) + if interface: + try: + ipaddrs = uci.get_all('network', interface, 'ipaddr') + except: + continue + if type(ipaddrs) == str: + ipaddrs = [(ipaddrs)] + for ipaddr in ipaddrs: + # skip bond management IPs + if ipaddr.startswith("127"): + continue + # strip mask to avoid duplicates + if ipaddr and '/' in ipaddr: + ipaddr = ipaddr.split('/')[0] + if ipaddr and ipaddr not in seen: + seen.add(ipaddr) + device = wan + if wan.startswith('@'): + device = uci.get('network', wan[1:], 'device', default='') + if device: + device = f'({device})' + ret.append({"device": f'{interface.replace("@","")} {device}', "ipaddr": ipaddr}) + # list on-line wans, search also for dynamic IPs + for device in ips.keys(): + if device in wans or device.startswith('pppoe'): + for ip in ips[device]: + # skip bond management IPs + if ip.startswith("127"): + continue + if ip not in seen: + seen.add(ip) + if device.startswith('pppoe-'): + interface = device.removeprefix('pppoe-') + else: + interface = get_interface_from_device(uci, device) + if interface: + ret.append({"device": f'{interface.replace("@","")} ({device})', "ipaddr": ip}) + + return sorted(ret, key=lambda k: k['ipaddr']) + + +def get_public_ip_addresses(ip_address=""): + """ + Return the public addresses associated to a private IP address. + + Arguments: + - ip_address -- the private IP address of a network interface. If not specified, the default network interface is used. + + Returns: + - a list of public addresses. Usually, the list contains only one element. + """ + try: + bindOption = "" + if ip_address: + bindOption = f" -b {ip_address}" + + cmd = f"/usr/bin/dig{bindOption} +short +time=1 myip.opendns.com @resolver1.opendns.com".split(" ") + output = subprocess.check_output(cmd, timeout=5) + public_ip_addresses = output.decode().strip().split('\n') + + if public_ip_addresses[0] == "": + # cannot retrieve public IP address, returning input IP address as fallback + if ip_address: + return [ip_address] + else: + return [] + else: + return public_ip_addresses + except Exception as e: + return [] + + +class ValidationError(ValueError): + def __init__(self, parameter, message="", value=""): + self.parameter = parameter + self.message = message + self.value = value diff --git a/packages/python3-nethsec/test.sh b/packages/python3-nethsec/test.sh new file mode 100755 index 000000000..b80c7ca6f --- /dev/null +++ b/packages/python3-nethsec/test.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# +# The container assumes the source code is mounted inside /app +# +IMAGE=python3-nethsec-test + +set -e + +# Always rebuild from the Containerfile: podman's layer cache makes this cheap +# when nothing changed, and a modified Containerfile can never be shadowed by a +# stale image. +podman build --force-rm --layers --jobs 0 --tag "${IMAGE}" "$(dirname "$0")" + +podman run --rm --tty --volume .:/app:Z "${IMAGE}" diff --git a/packages/python3-nethsec/tests/test_conntrack.py b/packages/python3-nethsec/tests/test_conntrack.py new file mode 100644 index 000000000..c1a5cf64f --- /dev/null +++ b/packages/python3-nethsec/tests/test_conntrack.py @@ -0,0 +1,455 @@ +from subprocess import CalledProcessError +from xml.etree import ElementTree + +import pytest +from pytest_mock import MockFixture + +from nethsec import conntrack + +conntrack_response = """ + + + + + 192.168.122.234 + 31.14.133.122 + + + 41692 + 123 + + + 1 + 76 + + + + + 31.14.133.122 + 192.168.122.234 + + + 123 + 41692 + + + 1 + 76 + + + + 47 + 16128 + 1 + 1905826093 + + + + + + 192.168.122.1 + 192.168.122.155 + + + + 2 + 168 + + + + + 192.168.122.155 + 192.168.122.1 + + + + 2 + 168 + + + + 29 + 16128 + 1 + 2860343346 + + + + + + 192.168.122.234 + 212.6.50.243 + + + 45538 + 123 + + + 1 + 76 + + + + + 212.6.50.243 + 192.168.122.234 + + + 123 + 45538 + + + 1 + 76 + + + + 48 + 16128 + 1 + 3555926027 + + + + + + 192.168.122.1 + 192.168.122.155 + + + 36578 + 22 + + + 73 + 7057 + + + + + 192.168.122.155 + 192.168.122.1 + + + 22 + 36578 + + + 54 + 10046 + + + + ESTABLISHED + 7440 + 16128 + 1 + 1285239412 + + + + + + + 192.168.122.234 + 93.49.6.247 + + + 50183 + 123 + + + 1 + 76 + + + + + 93.49.6.247 + 192.168.122.234 + + + 123 + 50183 + + + 1 + 76 + + + + 1 + 16128 + 1 + 3307146984 + + + + + + 192.168.122.1 + 192.168.122.255 + + + 57621 + 57621 + + + 1 + 72 + + + + + 192.168.122.255 + 192.168.122.1 + + + 57621 + 57621 + + + 0 + 0 + + + + 41 + 16128 + 1 + 3294787936 + + + + + + + 192.168.122.234 + 212.45.144.206 + + + 41960 + 123 + + + 1 + 76 + + + + + 212.45.144.206 + 192.168.122.234 + + + 123 + 41960 + + + 1 + 76 + + + + 48 + 16128 + 1 + 1915971940 + + + + + + 10.0.0.1 + 10.0.0.2 + + + 12345 + 80 + + + 5 + 500 + + + + + 10.0.0.2 + 10.0.0.1 + + + 80 + 12345 + + + 3 + 300 + + + + ESTABLISHED + 120 + 0 + 1 + 1111111111 + + + + + + + + + + 10.0.0.3 + 10.0.0.4 + + + 5353 + 53 + + + 1 + 60 + + + + + 10.0.0.4 + 10.0.0.3 + + + 53 + 5353 + + + 1 + 60 + + + + 30 + 0 + 1 + 2222222222 + + + + + + +""" + + +def test_connection_info(): + # get the first flow + flow = ElementTree.fromstring(conntrack_response).findall('flow')[0] + result = conntrack.__parse_connection_info(flow) + original = flow[0] + reply = flow[1] + assert result['source'] == '192.168.122.234' + assert result['destination'] == '31.14.133.122' + assert result['protocol'] == 'udp' + assert result['source_port'] == '41692' + assert result['destination_port'] == '123' + assert result['source_stats']['packets'] == 1 + assert result['source_stats']['bytes'] == 76 + assert result['destination_stats']['packets'] == 1 + assert result['destination_stats']['bytes'] == 76 + assert result['timeout'] == '47' + assert result['id'] == '1905826093' + flow = ElementTree.fromstring(conntrack_response).findall('flow')[1] + result = conntrack.__parse_connection_info(flow) + assert 'source_port' not in result + assert 'destination_port' not in result + + +def test_fetch_connection_list(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = conntrack_response + mocker.patch('subprocess.run', return_value=process_result) + result = conntrack.list_connections() + assert len(result) == 9 + + +def test_list_connections_label_filter(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = conntrack_response + subprocess_mock = mocker.patch('subprocess.run', return_value=process_result) + + # single label – '-l' flag appended with the label value + conntrack.list_connections(labels=['web']) + subprocess_mock.assert_called_with( + ['conntrack', '-L', '-o', 'labels,xml', '-l', 'web'], + capture_output=True, text=True + ) + + # multiple labels joined by comma + conntrack.list_connections(labels=['web', 'trusted']) + subprocess_mock.assert_called_with( + ['conntrack', '-L', '-o', 'labels,xml', '-l', 'web,trusted'], + capture_output=True, text=True + ) + + # no labels – no '-l' flag + conntrack.list_connections() + subprocess_mock.assert_called_with( + ['conntrack', '-L', '-o', 'labels,xml'], + capture_output=True, text=True + ) + + # empty list – no '-l' flag + conntrack.list_connections(labels=[]) + subprocess_mock.assert_called_with( + ['conntrack', '-L', '-o', 'labels,xml'], + capture_output=True, text=True + ) + + +def test_drop_connection(mocker: MockFixture): + connections_list = [ + { + 'protocol': 'udp', + 'source': '10.20.30.40', + 'destination': '1.1.1.1', + 'source_port': '123', + 'destination_port': '456', + 'id': '1234' + }, + { + 'protocol': 'icmp', + 'source': '1.1.1.1', + 'destination': '127.0.0.1', + 'id': '5678' + } + ] + mocker.patch('nethsec.conntrack.list_connections', return_value=connections_list) + command_line = mocker.patch('subprocess.run') + conntrack.drop_connection('1234') + command_line.assert_called_with([ + 'conntrack', '-D', '-p', 'udp', '-s', '10.20.30.40', '-d', '1.1.1.1', '--sport', '123', '--dport', '456' + ], check=True, capture_output=True) + conntrack.drop_connection('5678') + command_line.assert_called_with([ + 'conntrack', '-D', '-p', 'icmp', '-s', '1.1.1.1', '-d', '127.0.0.1' + ], check=True, capture_output=True) + with pytest.raises(ValueError) as e: + conntrack.drop_connection('9999') + + assert e.value.args[0] == "Connection with id 9999 not found." + with pytest.raises(RuntimeError) as e: + mocker.patch('subprocess.run', side_effect=CalledProcessError(1, 'conntrack')) + conntrack.drop_connection('1234') + + assert e.value.args[0].startswith("Error running command:") + + +def test_drop_all_connections(mocker: MockFixture): + command_line = mocker.patch('subprocess.run') + conntrack.drop_all_connections() + command_line.assert_called_with(['conntrack', '-F'], check=True, capture_output=True) + with pytest.raises(RuntimeError) as e: + mocker.patch('subprocess.run', side_effect=CalledProcessError(1, 'conntrack')) + conntrack.drop_all_connections() + + assert e.value.args[0].startswith("Error running command:") \ No newline at end of file diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py new file mode 100644 index 000000000..2e049dba2 --- /dev/null +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -0,0 +1,1067 @@ +import pathlib + +import pytest +from euci import EUci +from pytest_mock import MockFixture + +from nethsec import dpi +from nethsec.utils import ValidationError + +applications_file = """ +# Comment to avoid + +# another comment to avoid + +app:133:netify.netflix +app:10119:netify.linkedin +app:10552:netify.tesla +app:10195:netify.avira +app:10194:netify.sophos +app:10244:netify.bbc +app:10362:netify.hulu +app:10118:netify.lets-encrypt +app:199:netify.snapchat +dom:133:dualstack.apiproxy-device-prod-nlb-1-4d12762d4ba53e45.elb.eu-west-1.amazonaws.com +net:10852:208.118.237.0/24 +net:124:208.117.254.0/24 +nsd:-1:39:YXBwID09ICduZXRpZnkuc2lnbmFsJyAmJiBwcm90b2NvbF9pZCA9PSA3OCAmJiAob3RoZXJfcG9ydCA9PSA0NDMgfHwgbG9jYWxfcG9ydCA9PSA0NDMpOw== +nsd:-1:67:YXBwID09ICduZXRpZnkuZ29vZ2xlLWNoYXQnICYmIHByb3RvY29sX2lkICE9IDY3ICYmIChvdGhlcl9wb3J0ID09IDUyMjggfHwgbG9jYWxfcG9ydCA9PSA1MjI4KTs= +""" + +applications = { + 133: 'netify.netflix', + 10119: 'netify.linkedin', + 10552: 'netify.tesla', + 10195: 'netify.avira', + 10194: 'netify.sophos', + 10244: 'netify.bbc', + 10362: 'netify.hulu', + 10118: 'netify.lets-encrypt', + 199: 'netify.snapchat' +} + +protocols = { + 116: 'Warcraft3', + 117: 'LotusNotes', + 121: 'Dropbox', + 127: 'RPC', + 128: 'NetFlow', + 129: 'SFlow', + 130: 'HTTP/Connect' +} + +categories_file = """ +{ + "application_index": [ + [ + 3, + [ + 10362, + 10194, + 10552 + ] + ], + [ + 33, + [ + 133, + 10119, + 10195, + 10244 + ] + ], + [ + 20, + [ + 199 + ] + ] + ], + "application_tag_index": { + "first-category": 3, + "unknown": 33, + "last": 20 + }, + "protocol_index": [ + [ + 1, + [ + 121, + 127, + 128 + ] + ], + [ + 2, + [ + 116 + ] + ], + [ + 4, + [ + 129, + 130 + ] + ] + ], + "protocol_tag_index": { + "base": 1, + "games": 2, + "low": 4 + } +} +""" + +categories_file_old = """ +{ + "last_update": 1696950440, + "application_index": { + "first-category": [ + 10362, + 10194, + 10552 + ], + "unknown": [ + 133, + 10119, + 10195, + 10244 + ], + "last": [ + 199 + ] + }, + "protocol_index": { + "base": [ + 121, + 127, + 128 + ], + "games": [ + 116 + ], + "low": [ + 129, + 130 + ] + } +} +""" + +application_categories = { + 10362: { + 'name': 'first-category' + }, + 10194: { + 'name': 'first-category' + }, + 10552: { + 'name': 'first-category' + }, + 133: { + 'name': 'unknown' + }, + 10119: { + 'name': 'unknown' + }, + 10195: { + 'name': 'unknown' + }, + 10244: { + 'name': 'unknown' + }, + 199: { + 'name': 'last' + } +} + +protocol_categories = { + 121: { + 'name': 'base' + }, + 127: { + 'name': 'base' + }, + 128: { + 'name': 'base' + }, + 116: { + 'name': 'games' + }, + 129: { + 'name': 'low' + }, + 130: { + 'name': 'low' + } +} + +protocol_output = """ + 116: Warcraft3 + 117: LotusNotes + 121: Dropbox + 127: RPC + 128: NetFlow + 129: SFlow + 130: HTTP/Connect +""" + +dpi_minimal_db = """ +config main 'config' + option log_blocked '0' + option firewall_exemption '0' + option enabled '0' +""" + +dpi_db = """ +config main 'config' + option log_blocked '0' + option firewall_exemption '0' + option enabled '0' + list popular_filters 'netify.netflix' + list popular_filters 'netify.hulu' + list popular_filters 'netify.whatsapp' + list popular_filters 'netify.facebook' + list popular_filters 'netify.sophos' + list popular_filters 'HTTP/Connect' + list popular_filters 'Dropbox' + +config rule rule0 + option action 'block' + list application 'netify.linkedin' + list application 'netify.snapchat' + list protocol 'HTTP/Connect' + list source '192.168.100.1' + list source '192.168.100.2' + list source 'user:giacomo' + list source 'group:g1' + list category 'games' + option device 'eth0' + option enabled 1 + +config rule rule1 + option action 'block' + list application 'netify.tesla' + option device 'eth4' + option enabled 0 + list exemption '192.168.100.3' + +config rule rule2 + option action 'block' + option criteria 'local_ip == 192.168.100.22 && application == "netify.facebook";' + option enabled 1 + +config rule rule3 + option action 'video' + list protocol 'HTTP/Connect' + option device 'eth1' + option enabled 1 + +config exemption exemp1 + option criteria '192.168.1.1' + option description 'my host' + option enabled 1 +""" + +netifyd_config = """ +config netifyd + option enabled '1' + option autoconfig '0' + list internal_if 'br-lan' + list internal_if 'eth0' + list internal_if 'eth4' + list internal_if 'tunrw1' + list external_if 'eth2' + list external_if 'eth1' +""" + +network_config = """ +config interface 'RED_1' + option proto 'dhcp' + option device 'eth1' + option metric '1' + +config interface 'RED_2' + option proto 'dhcp' + option device 'eth2' + option metric '2' + +config interface 'RED_3' + option proto 'dhcp' + option device 'eth3' + option metric '3' + +config interface 'GREEN_1' + option proto 'static' + option device 'eth0' + option ipaddr '192.168.200.2' + option netmask '255.255.255.0' + option gateway '192.168.200.1' + +config interface 'GREEN_2' + option proto 'static' + option device 'eth4' + option ipaddr '10.0.3.1' + option netmask '255.255.255.0' + option gateway '10.0.3.0' + +config interface 'GREEN_3' + option proto 'static' + option device 'br_lan' + option ipaddr '10.1.1.1' + option netmask '255.255.255.0' + option gateway '10.1.1.0' +""" + +firewall_config = """ +config zone 'ns_lan' + option name 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list network 'GREEN_1' + list network 'GREEN_2' + +config zone 'ns_wan' + option name 'wan' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + list network 'RED_1' + list network 'RED_2' + list network 'RED_3' + +config zone 'ns_guests' + option name 'guests' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + list network 'GREEN_1' + +config zone 'ns_empty' + option name 'guests' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' +""" + + +@pytest.fixture +def e_uci(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + with conf_dir.joinpath('dpi').open('w') as fp: + fp.write(dpi_minimal_db) + with conf_dir.joinpath('network').open('w') as fp: + fp.write(network_config) + with conf_dir.joinpath('firewall').open('w') as fp: + fp.write(firewall_config) + return EUci(confdir=conf_dir.as_posix(), savedir=save_dir.as_posix()) + + +@pytest.fixture +def e_uci_with_data(e_uci: EUci): + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(dpi_db) + with pathlib.Path(e_uci.confdir()).joinpath('netifyd').open('w') as fp: + fp.write(netifyd_config) + return e_uci + + +@pytest.fixture +def mock_load(mocker): + mocker.patch('nethsec.dpi.__load_applications', return_value=applications) + mocker.patch('nethsec.dpi.__load_application_categories', return_value=application_categories) + mocker.patch('nethsec.dpi.load_protocols', return_value=protocols) + mocker.patch('nethsec.dpi.__load_protocol_categories', return_value=protocol_categories) + + +def test_load_applications(mocker: MockFixture): + mocker.patch('builtins.open', mocker.mock_open(read_data=applications_file)) + assert dpi.__load_applications() == applications + + +@pytest.mark.parametrize('data', [categories_file, categories_file_old]) +def test_load_application_categories(mocker: MockFixture, data: str): + mocker.patch('builtins.open', mocker.mock_open(read_data=data)) + assert dpi.__load_application_categories() == application_categories + + +@pytest.mark.parametrize('data', [categories_file, categories_file_old]) +def test_load_protocol_categories(mocker: MockFixture, data: str): + mocker.patch('builtins.open', mocker.mock_open(read_data=data)) + assert dpi.__load_protocol_categories() == protocol_categories + + +def test_load_protocols(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = bytes(protocol_output, 'utf-8') + mocker.patch('subprocess.run', return_value=process_result) + assert dpi.load_protocols() == protocols + + +@pytest.mark.parametrize('search', [None, '']) +def test_index_applications(mock_load, search): + assert dpi.list_applications(search) == { + 'data': [ + { + 'id': 133, + 'name': 'netify.netflix', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10119, + 'name': 'netify.linkedin', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10195, + 'name': 'netify.avira', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10194, + 'name': 'netify.sophos', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10244, + 'name': 'netify.bbc', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10362, + 'name': 'netify.hulu', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10118, + 'name': 'netify.lets-encrypt', + 'type': 'application' + }, + { + 'id': 199, + 'name': 'netify.snapchat', + 'type': 'application', + 'category': { + 'name': 'last' + } + }, + { + 'id': 116, + 'name': 'Warcraft3', + 'type': 'protocol', + 'category': { + 'name': 'games' + } + }, + { + 'id': 117, + 'name': 'LotusNotes', + 'type': 'protocol' + }, + { + 'id': 121, + 'name': 'Dropbox', + 'type': 'protocol', + 'category': { + 'name': 'base' + } + }, + { + 'id': 127, + 'name': 'RPC', + 'type': 'protocol', + 'category': { + 'name': 'base' + } + }, + { + 'id': 128, + 'name': 'NetFlow', + 'type': 'protocol', + 'category': { + 'name': 'base' + } + }, + { + 'id': 129, + 'name': 'SFlow', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ], + 'meta': { + 'last_page': 1, + 'total': 16 + } + } + + +def test_index_applications_search(mock_load): + assert dpi.list_applications(search='l') == { + 'data': [ + { + 'id': 133, + 'name': 'netify.netflix', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10119, + 'name': 'netify.linkedin', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10362, + 'name': 'netify.hulu', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10118, + 'name': 'netify.lets-encrypt', + 'type': 'application' + }, + { + 'id': 199, + 'name': 'netify.snapchat', + 'type': 'application', + 'category': { + 'name': 'last' + } + }, + { + 'id': 117, + 'name': 'LotusNotes', + 'type': 'protocol' + }, + { + 'id': 128, + 'name': 'NetFlow', + 'type': 'protocol', + 'category': { + 'name': 'base' + } + }, + { + 'id': 129, + 'name': 'SFlow', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ], + 'meta': { + 'last_page': 1, + 'total': 10 + } + } + + +def test_index_applications_paginate(mock_load): + assert dpi.list_applications(limit=2, page=2) == { + 'data': [ + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10195, + 'name': 'netify.avira', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + } + ], + 'meta': { + 'last_page': 8, + 'total': 16 + } + } + assert dpi.list_applications(limit=2, page=3) == { + 'data': [ + { + 'id': 10194, + 'name': 'netify.sophos', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + }, + { + 'id': 10244, + 'name': 'netify.bbc', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + } + ], + 'meta': { + 'last_page': 8, + 'total': 16 + } + } + + +def test_list_empty_rules(e_uci, mock_load): + assert dpi.list_rules(e_uci) == [] + + +def test_list_rules(e_uci_with_data, mock_load): + assert dpi.list_rules(e_uci_with_data) == [ + { + 'config-name': 'rule0', + 'enabled': True, + 'interface': 'GREEN_1', + 'device': 'eth0', + 'action': 'block', + 'criteria': [ + { + 'id': 10119, + 'name': 'netify.linkedin', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 199, + 'name': 'netify.snapchat', + 'type': 'application', + 'category': { + 'name': 'last' + } + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + }, + { + 'config-name': 'rule1', + 'enabled': False, + 'interface': 'GREEN_2', + 'device': 'eth4', + 'action': 'block', + 'criteria': [ + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + } + ] + }, + { + 'config-name': 'rule3', + 'enabled': True, + 'interface': 'RED_1', + 'device': 'eth1', + 'action': 'video', + 'criteria': [ + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + } + ] + + +def test_store_rule(e_uci, mock_load): + rule_created = dpi.add_rule(e_uci, True, 'eth0', 'best_effort', + ['netify.linkedin', 'netify.avira', 'netify.netflix'], ['LotusNotes', 'SFlow']) + assert dpi.list_rules(e_uci) == [ + { + 'config-name': rule_created, + 'enabled': True, + 'interface': 'GREEN_1', + 'device': 'eth0', + 'action': 'best_effort', + 'criteria': [ + { + 'id': 10119, + 'name': 'netify.linkedin', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 10195, + 'name': 'netify.avira', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 133, + 'name': 'netify.netflix', + 'type': 'application', + 'category': { + 'name': 'unknown' + } + }, + { + 'id': 117, + 'name': 'LotusNotes', + 'type': 'protocol', + }, + { + 'id': 129, + 'name': 'SFlow', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + } + ] + assert(e_uci.get("dpi", "config", "enabled") == "1") + + +def test_delete_rule(e_uci_with_data, mock_load): + dpi.delete_rule(e_uci_with_data, 'rule1') + dpi.delete_rule(e_uci_with_data, 'rule0') + assert dpi.list_rules(e_uci_with_data) == [ + { + 'config-name': 'rule3', + 'enabled': True, + 'interface': 'RED_1', + 'device': 'eth1', + 'action': 'video', + 'criteria': [ + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + } + ] + dpi.delete_rule(e_uci_with_data, 'rule2') + dpi.delete_rule(e_uci_with_data, 'rule3') + assert(e_uci_with_data.get("dpi", "config", "enabled") == "0") + + +def test_edit_rule(e_uci_with_data, mock_load): + dpi.edit_rule(e_uci_with_data, 'rule0', False, 'eth0', 'voice', [], + ['HTTP/Connect', 'LotusNotes']) + assert dpi.list_rules(e_uci_with_data) == [ + { + 'config-name': 'rule0', + 'enabled': False, + 'interface': 'GREEN_1', + 'device': 'eth0', + 'action': 'voice', + 'criteria': [ + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + }, + { + 'id': 117, + 'name': 'LotusNotes', + 'type': 'protocol', + } + ] + }, + { + 'config-name': 'rule1', + 'enabled': False, + 'interface': 'GREEN_2', + 'device': 'eth4', + 'action': 'block', + 'criteria': [ + { + 'id': 10552, + 'name': 'netify.tesla', + 'type': 'application', + 'category': { + 'name': 'first-category' + } + } + ] + }, + { + 'config-name': 'rule3', + 'enabled': True, + 'interface': 'RED_1', + 'device': 'eth1', + 'action': 'video', + 'criteria': [ + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + } + } + ] + } + ] + + +def test_edit_rule_with_missing_rule(e_uci): + with pytest.raises(ValidationError) as err: + dpi.edit_rule(e_uci, 'rule0', False, 'eth0', 'block', [], []) + + assert err.value.args[0] == 'config-name' + assert err.value.args[1] == 'invalid' + assert err.value.args[2] == 'rule0' + + +def test_list_interfaces(e_uci_with_data): + assert dpi.list_devices(e_uci_with_data).index({ + 'interface': 'GREEN_1', + 'device': 'eth0' + }) != -1 + assert dpi.list_devices(e_uci_with_data).index({ + 'interface': 'GREEN_2', + 'device': 'eth4' + }) != -1 + with pytest.raises(ValueError): + assert dpi.list_devices(e_uci_with_data).index({ + 'interface': 'RED_1', + 'device': 'eth1' + }) + + +def test_list_popular(e_uci_with_data, mock_load): + assert dpi.list_popular(e_uci_with_data) == { + 'data': [ + { + 'id': 133, + 'name': 'netify.netflix', + 'type': 'application', + 'category': { + 'name': 'unknown' + }, + 'missing': False + }, + { + 'id': 10362, + 'name': 'netify.hulu', + 'type': 'application', + 'category': { + 'name': 'first-category' + }, + 'missing': False + }, + { + 'name': 'netify.whatsapp', + 'missing': True + }, + { + 'name': 'netify.facebook', + 'missing': True + }, + { + 'id': 10194, + 'name': 'netify.sophos', + 'type': 'application', + 'category': { + 'name': 'first-category' + }, + 'missing': False + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + }, + 'missing': False + }, + { + 'id': 121, + 'name': 'Dropbox', + 'type': 'protocol', + 'category': { + 'name': 'base' + }, + 'missing': False + } + ], + 'meta': { + 'last_page': 1, + 'total': 7 + } + } + + +def test_list_popular_with_limits(e_uci_with_data, mock_load): + assert dpi.list_popular(e_uci_with_data, limit=2, page=2) == { + 'data': [ + { + 'name': 'netify.whatsapp', + 'missing': True + }, + { + 'name': 'netify.facebook', + 'missing': True + } + ], + 'meta': { + 'last_page': 4, + 'total': 7 + } + } + assert dpi.list_popular(e_uci_with_data, limit=2, page=3) == { + 'data': [ + { + 'id': 10194, + 'name': 'netify.sophos', + 'type': 'application', + 'category': { + 'name': 'first-category' + }, + 'missing': False + }, + { + 'id': 130, + 'name': 'HTTP/Connect', + 'type': 'protocol', + 'category': { + 'name': 'low' + }, + 'missing': False + } + ], + 'meta': { + 'last_page': 4, + 'total': 7 + } + } + +def test_list_exemptions(e_uci_with_data): + assert dpi.list_exemptions(e_uci_with_data) == [ + { + 'config-name': 'exemp1', + 'enabled': True, + 'criteria': '192.168.1.1', + 'description': 'my host', + } + ] + +def test_add_exemption(e_uci): + ex_created = dpi.add_exemption(e_uci, "192.168.2.2", 'my host2', True) + assert dpi.list_exemptions(e_uci) == [ + { + 'config-name': ex_created, + 'enabled': True, + 'criteria': "192.168.2.2", + 'description': 'my host2', + } + ] + with pytest.raises(ValidationError) as err: + dpi.add_exemption(e_uci, "192.168.2.2", 'duplicated', True) + +def test_edit_exemption(e_uci_with_data): + dpi.edit_exemption(e_uci_with_data, 'exemp1', '192.168.1.3', 'my host 3', False) + assert dpi.list_exemptions(e_uci_with_data) == [ + { + 'config-name': 'exemp1', + 'enabled': False, + 'criteria': "192.168.1.3", + 'description': 'my host 3', + } + ] + +def test_delete_exemption(e_uci): + dpi.delete_exemption(e_uci, 'exemp1') + assert dpi.list_exemptions(e_uci) == [] diff --git a/packages/python3-nethsec/tests/test_firewall.py b/packages/python3-nethsec/tests/test_firewall.py new file mode 100644 index 000000000..0920bcb31 --- /dev/null +++ b/packages/python3-nethsec/tests/test_firewall.py @@ -0,0 +1,1262 @@ +from nethsec import utils +from nethsec.utils import ValidationError +import pytest +from euci import EUci, UciExceptionNotFound + +from nethsec import firewall, objects +from pytest_mock import MockFixture +from unittest.mock import MagicMock, patch +import pathlib +firewall_db = """ +config zone lan1 + option name 'lan' + list network 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list device 'eth0' + +config zone grey + option name 'grey' + list network 'grey1' + option input 'DROP' + option output 'ACCEPT' + option forward 'DROP' + list device 'eth1' + +config zone orange + option name 'orange' + list network 'orange1' + option input 'DROP' + option output 'DROP' + option forward 'DROP' + +config zone wan1f + option name 'wan' + list network 'wan' + list network 'wan6' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + +config zone with_log + option name 'with_log' + option input 'DROP' + option output 'DROP' + option forward 'DROP' + option log '1' + +config forwarding fw1 + option src 'lan' + option dest 'wan' + +config rule 'v6rule' + option name 'Allow-DHCPv6' + option src 'wan' + option proto 'udp' + option dest_port '546' + option family 'ipv6' + option target 'ACCEPT' + +config rule 'manrule' + option name 'Not-Automated' + option src 'wan' + option proto 'tcp' + option dest_port '1234' + option family 'ipv4' + option target 'ACCEPT' + +config rule 'f1' + option name 'r1' + option dest 'wan' + option dest_port '22' + option target 'ACCEPT' + option src 'lan' + list src_ip '192.168.100.1' + list src_ip '192.168.100.238' + list dest_ip '192.168.122.1' + list dest_ip '192.168.122.49' + option log '1' + +config rule 'o1' + option name 'output1' + list dest_ip '192.168.100.1' + option target 'ACCEPT' + option dest 'wan' + +config rule 'i1' + option name 'Allow-OpenVPNRW1' + option src 'wan' + option dest_port '1194' + option proto 'udp' + option target 'ACCEPT' + list ns_tag 'automated' + option ns_link 'openvpn/ns_roadwarrior1' + +config nat + option name 'source_NAT1_vpm' + option src '*' + option src_ip '192.168.55.0/24' + option dest_ip '10.20.30.0/24' + option target 'SNAT' + option snat_ip '10.44.44.1' + list proto 'all' + +config nat + option name 'masquerade' + list proto 'all' + option src 'lan' + option src_ip '192.168.1.0/24' + option dest_ip '10.88.88.0/24' + option target 'MASQUERADE' + +config nat + option name 'cdn_via_router' + list proto 'all' + option src 'lan' + option src_ip '192.168.1.0/24' + option dest_ip '192.168.50.0/24' + option target 'ACCEPT' + +config nat + option name 'SNAT_NSEC7_style' + list proto 'all' + option src 'wan' + option src_ip '192.168.1.44' + option target 'SNAT' + option snat_ip '10.20.30.5' + +config redirect 'redirect3' + option ns_src '' + option ipset 'redirect3_ipset' + +config ipset 'redirect3_ipset' + option name 'redirect1_ipset' + option match 'src_net' + option enabled '1' + list entry '6.7.8.9' + +config redirect 'redirect4' + option ns_src '' + +config rule 'r1' + option name 'r1' + option ns_dst 'dhcp/ns_8dcab636' + +config rule 'r2' + option name 'r2' + option ns_dst '' + +config rule 'r3' + option name 'r3' + option ns_src '' + +config rule 'r4' + option ns_dst '' + option ns_src '' + +config redirect 'ns_59eea167' + option src 'wan' + option target 'DNAT' + option dest_ip '192.168.1.22' + option dest_port '4466' + option enabled '1' + option log '1' + option name 'pf1' + list proto 'tcp' + list proto 'udp' + option reflection '0' + option src_dport '4455' +""" + +network_db = """ +config interface lan + option device 'vnet3' + +config interface lan2 + option device 'vnet4' + +config interface grey1 + option device 'vnet5' + +config interface orange1 + option device 'vnet6' + +config interface 'lan6' + option device 'br-lan' + option proto 'dhcpv6' + +config device 'vlan6' + option type '8021ad' + option ifname 'eth2' + option vid '1' + option name 'eth2.1' + option ipv6 '1' + +config interface 'wan6' + option device eth1 + option proto static + option ip6addr 2001:db80::2/64 + option ip6gw 2001:db80::1 + option ip6prefix 2001:db80:1::/48 + option dns 2001:db80::1 + +config interface 'wan6b' + option device eth1 + option proto vtiv6 + +config interface 'wan6c' + option device eth44 + option ipv6 1 + +config interface 'bond1' + option proto 'bonding' + option ipaddr '10.0.0.22' + option netmask '255.255.255.0' + list slaves 'eth3' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + option all_slaves_active '0' + option link_monitoring 'off' + +config interface 'lan' + option device 'br-lan' + option proto 'static' + option ipaddr '192.168.100.238' + option netmask '255.255.255.0' + +config interface 'al_lan' + option proto 'static' + option device '@lan' + list ipaddr '10.0.0.1/24' +""" + +templates_db = """ +# Service groups +config template_service_group 'ns_web_secure' + option name 'Secure web navigation' + list services '80/tcp/HTTP' + list services '443/tcp/HTTP Secure' + list services '53/udp/DNS' + +# Blue zone +config template_zone 'ns_blue' + option name 'blue' + option forward 'DROP' + option input 'DROP' + option output 'ACCEPT' + option ns_description 'Guest network with Internet access' + list forwardings 'ns_blue2wan' + list forwardings 'ns_blue2lan' + # requires network option + +config template_forwarding 'ns_blue2wan' + option src 'blue' + option dest 'wan' + +config template_forwarding 'ns_blue2lan' + option src 'blue' + option dest 'lan' + +# Default rule +config template_rule 'ns_test_rule' + option name 'Test-rule' + option src 'wan' + option dest 'blue' + option dest_port '__PORT__' + option proto '__PROTO__' + option target 'ACCEPT' + option enabled '1' + +# IPv6 rules + +config template_rule 'ip6_dhcp' + option name 'Allow-DHCPv6' + option src 'wan' + option proto 'udp' + option dest_port '546' + option family 'ipv6' + option target 'ACCEPT' + +config template_rule 'ip6_mld' + option name 'Allow-MLD' + option src 'wan' + option proto 'icmp' + option src_ip 'fe80::/10' + list icmp_type '130/0' + list icmp_type '131/0' + list icmp_type '132/0' + list icmp_type '143/0' + option family 'ipv6' + option target 'ACCEPT' + +config template_rule 'ip6_icmp' + option name 'Allow-ICMPv6-Input' + option src 'wan' + option proto 'icmp' + list icmp_type 'echo-request' + list icmp_type 'echo-reply' + list icmp_type 'destination-unreachable' + list icmp_type 'packet-too-big' + list icmp_type 'time-exceeded' + list icmp_type 'bad-header' + list icmp_type 'unknown-header-type' + list icmp_type 'router-solicitation' + list icmp_type 'neighbour-solicitation' + list icmp_type 'router-advertisement' + list icmp_type 'neighbour-advertisement' + option limit '1000/sec' + option family 'ipv6' + option target 'ACCEPT' + +config template_rule 'ip6_icmp_forward' + option name 'Allow-ICMPv6-Forward' + option src 'wan' + option dest '*' + option proto 'icmp' + list icmp_type 'echo-request' + list icmp_type 'echo-reply' + list icmp_type 'destination-unreachable' + list icmp_type 'packet-too-big' + list icmp_type 'time-exceeded' + list icmp_type 'bad-header' + list icmp_type 'unknown-header-type' + option limit '1000/sec' + option family 'ipv6' + option target 'ACCEPT' + +""" + +zone_testing_db = """ +config zone 'ns_lan' + option name 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list network 'GREEN_1' + list network 'al_lan' + +config zone 'ns_wan' + option name 'wan' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + option log '1' + list network 'wan6' + list network 'RED_2' + list network 'RED_3' + list network 'RED_1' + +config zone 'ns_guests' + option name 'guests' + option input 'DROP' + option forward 'DROP' + option output 'ACCEPT' + +config forwarding + option src 'lan' + option dest 'wan' + +config forwarding 'ns_guests2wan' + option src 'guests' + option dest 'wan' + +config forwarding 'ns_lan2guests' + option src 'lan' + option dest 'guests' +""" + +dhcp_db = """ +config domain + option name 'test.name.org' + option ip '192.168.100.1' + +config host + option name 'test2.giacomo.org' + option mac '5c:87:9c:fa:69:5b' + option ip '192.168.100.2' + +config domain + option name 'test3.test.org' + option ip 'ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543' + +config domain 'ns_dhcp1' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' +""" + +netmap_db = """ +config rule + option name 'source_nat1' + option dest '10.50.50.0/24' + list device_in 'eth0' + list device_in 'eth1' + list device_out 'tunrw' + option map_from '192.168.1.0/24' + option map_to '192.168.57.0/24' + +config rule + option name 'dest_nat1' + option src '10.50.50.0/24' + option map_from '192.168.1.0/24' + option map_to '192.168.57.0/24' +""" + +services_file = """ +ftp 21/tcp +ssh 22/tcp +ssh 22/udp +time 37/udp +www 80/tcp http +kerberos 88/tcp kerberos5 krb5 kerberos-sec +kerberos 88/udp kerberos5 krb5 kerberos-sec +""" + +lease_file = """ +1704890657 9c:3d:cf:ea:94:9b 192.168.1.70 * * +1704885018 98:ed:5c:8c:73:2a 192.168.1.228 test1 * +1704874398 ac:57:26:00:24:8c 192.168.1.219 test2 01:dc:57:26:00:25:8c +""" + +objects_db = """ +""" + +user_db = """ +config user 'ns_user1' + option name "john" + option database "main" + option label "John Doe" + option openvpn_ipaddr "10.10.10.22" +""" + +mwan3_db = """ +""" + +dpi_db = """ +""" + +# Setup fake ip command output +ip_json='[{"ifindex":9,"ifname":"vnet3","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"virbr2","operstate":"UNKNOWN","group":"default","txqlen":1000,"link_type":"ether","address":"fe:62:31:19:0b:29","broadcast":"ff:ff:ff:ff:ff:ff","addr_info":[{"family":"inet6","local":"fe80::fc62:31ff:fe19:b29","prefixlen":64,"scope":"link","valid_life_time":4294967295,"preferred_life_time":4294967295}]},{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":3,"ifname":"eth1","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":4,"ifname":"eth2","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:75:1c:c1","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":5,"ifname":"eth3","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":6,"ifname":"ifb-dns","flags":["BROADCAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":32,"link_type":"ether","address":"72:79:65:12:07:07","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":7,"ifname":"br-lan","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":9,"ifname":"bond-bond1","flags":["BROADCAST","MULTICAST","MASTER","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":8,"ifname":"tuntunsubnet","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":500,"link_type":"none"}, {"ifindex":69,"ifname":"pppoe-w1","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1492,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":3,"link_type":"ppp"},{"ifindex":20,"link":"eth1","ifname":"eth1.4","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"}]' +mock_ip_stdout = MagicMock() +mock_ip_stdout.configure_mock(**{"stdout": ip_json}) + +@pytest.fixture +def u(tmp_path: pathlib.Path) -> EUci: + # setup fake dbs + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + fp.write(zone_testing_db) + with tmp_path.joinpath('network').open('w') as fp: + fp.write(network_db) + with tmp_path.joinpath('templates').open('w') as fp: + fp.write(templates_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('netmap').open('w') as fp: + fp.write(netmap_db) + with tmp_path.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with tmp_path.joinpath('users').open('w') as fp: + fp.write(user_db) + with tmp_path.joinpath('mwan3').open('w') as fp: + fp.write(mwan3_db) + with tmp_path.joinpath('dpi').open('w') as fp: + fp.write(dpi_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_add_interface_to_zone(u): + z1 = firewall.add_interface_to_zone(u, "interface1", "lan") + assert z1 == 'lan1' + assert 'interface1' in u.get_all('firewall', 'lan1', 'network') + assert firewall.add_interface_to_zone(u, "interface1", "blue") == None + z1 = firewall.add_interface_to_zone(u, "interface2", "lan") + assert 'interface2' in u.get_all('firewall', 'lan1', 'network') + +def test_remove_interface_from_zone(u): + z1 = firewall.remove_interface_from_zone(u, 'interface1', "lan") + assert(not 'interface1' in u.get_all('firewall', 'lan1', 'network')) + +def test_add_device_to_zone(u): + z1 = firewall.add_device_to_zone(u, "vnet1", "lan") + assert z1 == 'lan1' + assert 'vnet1' in u.get_all('firewall', 'lan1', 'device') + assert firewall.add_device_to_zone(u, "vnet1", "blue") == None + +def test_add_device_to_lan(u): + assert firewall.add_device_to_lan(u, "vnet1") == 'lan1' + assert 'vnet1' in u.get_all('firewall', 'lan1', 'device') + +def test_add_device_to_wan(u): + assert firewall.add_device_to_wan(u, "vnet2") == 'wan1f' + assert 'vnet2' in u.get_all('firewall', 'wan1f', 'device') + +def test_remove_device_from_zone(u): + firewall.remove_device_from_zone(u, 'vnet1', "lan") + assert(not 'vnet2' in u.get_all('firewall', 'lan1', 'device')) + +def test_add_service(u): + rule = firewall.add_service(u, "my_service", "443", "tcp", "nginx/_lan") + assert rule is not None + assert rule == "ns_allow_my_service" + assert u.get('firewall', rule, 'dest_port') == '443' + assert u.get('firewall', rule, 'proto') == 'tcp' + assert u.get('firewall', rule, 'src') == 'wan' + assert u.get('firewall', rule, 'name') == 'Allow-my_service' + assert u.get('firewall', rule, 'ns_link') == "nginx/_lan" + assert u.get('firewall', rule, 'ns_tag') == "automated" + + assert firewall.add_service(u, "my-service2", "456", ["tcp", "udp"]) == "ns_allow_my_service2" + assert u.get_all('firewall', 'ns_allow_my_service2', 'proto') == ("tcp", "udp") + +def test_block_service(u): + firewall.add_service(u, "my-service", "443", "tcp") + assert firewall.remove_service(u, "my-service") == "ns_allow_my_service" + with pytest.raises(UciExceptionNotFound): + u.get('firewall', 'ns_allow_my_service') + +def test_disable_service(u): + firewall.add_service(u, "my-service", "443", "tcp") + assert firewall.disable_service(u, "my-service") == "ns_allow_my_service" + assert u.get("firewall", "ns_allow_my_service", "enabled") == "0" + assert firewall.disable_service(u, "non-existing") == None + +def test_enable_service(u): + firewall.add_service(u, "my-service", "1234", "tcp") + firewall.disable_service(u, "my-service") + assert firewall.enable_service(u, "my-service") == "ns_allow_my_service" + assert u.get("firewall", "ns_allow_my_service", "enabled") + assert firewall.enable_service(u, "non-existing") == None + +def test_add_vpn_interface(u): + assert firewall.add_vpn_interface(u, 'test!vpn', 'tuntest') == 'test_vpn' + assert u.get('network', 'test_vpn') == 'interface' + assert u.get('network', 'test_vpn', 'proto') == 'none' + assert u.get('network', 'test_vpn', 'device') == 'tuntest' + assert u.get('network', 'test_vpn', 'ns_tag') == 'automated' + i = firewall.add_vpn_interface(u, 'p2p', 'ppp10', 'torrent/server1') + assert u.get('network', 'p2p', 'ns_link') == 'torrent/server1' + +def test_add_trusted_zone(u): + assert firewall.add_trusted_zone(u, 'toolongnameforzone') == (None, None) + + (zone, forwardings) = firewall.add_trusted_zone(u, 'mytrusted') + assert zone is not None + assert len(forwardings) == 3 + assert u.get("firewall", zone) == "zone" + assert u.get("firewall", zone, 'name') == "mytrusted" + assert u.get("firewall", zone, 'input') == "ACCEPT" + assert u.get("firewall", zone, 'output') == "ACCEPT" + assert u.get("firewall", zone, 'forward') == "REJECT" + assert u.get("firewall", forwardings[0]) == "forwarding" + assert u.get("firewall", forwardings[0], 'src') == "mytrusted" + assert u.get("firewall", forwardings[0], 'dest') == "lan" + assert u.get("firewall", forwardings[0], 'ns_tag') == "automated" + assert u.get("firewall", forwardings[1]) == "forwarding" + assert u.get("firewall", forwardings[1], 'src') == "lan" + assert u.get("firewall", forwardings[1], 'dest') == "mytrusted" + assert u.get("firewall", forwardings[1], 'ns_tag') == "automated" + assert u.get("firewall", forwardings[2]) == "forwarding" + assert u.get("firewall", forwardings[2], 'src') == "mytrusted" + assert u.get("firewall", forwardings[2], 'dest') == "wan" + assert u.get("firewall", forwardings[2], 'ns_tag') == "automated" + + link = 'openvpn/instance1' + (zone, forwardings) = firewall.add_trusted_zone(u, 'mylinked', link=link) + assert u.get("firewall", zone, 'ns_link') == link + assert u.get("firewall", forwardings[0], 'ns_link') == link + assert u.get("firewall", forwardings[1], 'ns_link') == link + assert u.get("firewall", forwardings[2], 'ns_link') == link + +def test_duplicated_add_trusted_zone(u): + (zone, forwardings) = firewall.add_trusted_zone(u, 'mytrusted') + assert zone is None + assert forwardings is None + + trusted = 0 + for s in u.get_all('firewall'): + if u.get('firewall', s) == 'forwarding': + if u.get('firewall', s, 'src', default='') == "mytrusted" and u.get('firewall', s, 'dest', default='') == "lan": + trusted = trusted + 1 + assert trusted == 1 + +def test_add_trusted_zone_with_networks(u): + interface = firewall.add_vpn_interface(u, 'testvpn2', 'tuntest2') + zone, forwardings = firewall.add_trusted_zone(u, 'mytrusted2', list(interface)) + assert u.get_all("firewall", zone, 'network') == tuple(interface) + +def test_apply(): + # Already tested in pyuci + assert 1 + +def test_add_template_rule(u): + rule = firewall.add_template_rule(u, 'ns_test_rule', 'tcp', '443', 'test1/key1') + assert u.get("firewall", rule) == "rule" + assert u.get("firewall", rule, "proto") == "tcp" + assert u.get("firewall", rule, "name") == "Test-rule" + assert u.get("firewall", rule, "src") == "wan" + assert u.get("firewall", rule, "dest") == "blue" + assert u.get("firewall", rule, "dest_port") == "443" + assert u.get("firewall", rule, "enabled") == "1" + assert u.get("firewall", rule, "target") == "ACCEPT" + assert u.get("firewall", rule, "ns_tag") == "automated" + assert u.get("firewall", rule, "ns_link") == "test1/key1" + + rule = firewall.add_template_rule(u, 'ip6_dhcp') + assert u.get("firewall", rule) == "rule" + assert u.get("firewall", rule, "proto") == "udp" + assert u.get("firewall", rule, "name") == "Allow-DHCPv6" + assert u.get("firewall", rule, "src") == "wan" + assert u.get("firewall", rule, "dest_port") == "546" + assert u.get("firewall", rule, "family") == "ipv6" + assert u.get("firewall", rule, "target") == "ACCEPT" + assert u.get("firewall", rule, "ns_tag") == "automated" + +def test_add_template_zone(u): + (zone, forwardings) = firewall.add_template_zone(u, 'ns_blue', ["lan", "lan2"], link="mydb/mykey" ) + assert zone is not None + assert u.get("firewall", zone) == "zone" + assert u.get("firewall", zone, "name") == "blue" + assert u.get("firewall", zone, "forward") == "DROP" + assert u.get("firewall", zone, "input") == "DROP" + assert u.get("firewall", zone, "output") == "ACCEPT" + assert u.get("firewall", zone, "ns_description") == "Guest network with Internet access" + assert u.get("firewall", zone, "ns_tag") == "automated" + assert u.get("firewall", zone, "ns_link") == "mydb/mykey" + assert "lan" in u.get("firewall", zone, "network", list=True) + assert "lan2" in u.get("firewall", zone, "network", list=True) + assert len(forwardings) == 2 + for f in forwardings: + assert u.get("firewall", f) == "forwarding" + assert u.get("firewall", f, "ns_tag") == "automated" + assert u.get("firewall", f, "src") == "blue" or u.get("firewall", f, "dest") == "blue" + assert u.get("firewall", f, "ns_link", default='') == "mydb/mykey" + (zone, forwardings) = firewall.add_template_zone(u, 'ns_blue') + assert zone is None + assert forwardings is None + +def test_add_template_service_group(u): + sections = firewall.add_template_service_group(u, "ns_web_secure") + assert len(sections) == 2 + assert u.get("firewall", sections[0]) == "rule" + assert u.get("firewall", sections[0], "src") == "lan" + assert u.get("firewall", sections[0], "dest") == "wan" + assert u.get("firewall", sections[0], "proto") == "tcp" + assert u.get("firewall", sections[0], "dest_port") == "80,443" + assert u.get("firewall", sections[0], "ns_tag") == "automated" + + assert u.get("firewall", sections[1]) == "rule" + assert u.get("firewall", sections[1], "src") == "lan" + assert u.get("firewall", sections[1], "dest") == "wan" + assert u.get("firewall", sections[1], "proto") == "udp" + assert u.get("firewall", sections[1], "dest_port") == "53" + assert u.get("firewall", sections[1], "ns_tag") == "automated" + + sections = firewall.add_template_service_group(u, "ns_web_secure", "grey", "orange") + assert u.get("firewall", sections[0], "src") == "grey" + assert u.get("firewall", sections[0], "dest") == "orange" + assert u.get("firewall", sections[0], "proto") == "tcp" + assert u.get("firewall", sections[1], "proto") == "udp" + + sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link="db/mykey") + assert u.get("firewall", sections[0], "ns_link") == "db/mykey" + assert u.get("firewall", sections[1], "ns_link") == "db/mykey" + +def test_get_all_linked(u): + link = "mytestdb/mykey" + sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) + rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) + interface = firewall.add_vpn_interface(u, 'p2p', 'ppp10', link=link) + (zone, forwardings) = firewall.add_trusted_zone(u, 'mylinked2', link=link) + linked = firewall.get_all_linked(u, link) + for s in sections: + assert s in linked['firewall'] + assert rule in linked['firewall'] + assert zone in linked['firewall'] + for f in forwardings: + assert f in linked['firewall'] + assert interface in linked['network'] + + +def test_disable_linked_rules(u): + link = "mytestdb/mykey" + sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) + rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) + interface = firewall.add_vpn_interface(u, 'p2p', 'ppp10', link=link) + (zone, forwardings) = firewall.add_trusted_zone(u, 'mylinked4', link=link) + disabled = firewall.disable_linked_rules(u, link) + for s in sections: + assert u.get("firewall", s, "enabled") == "0" + assert s in disabled + assert u.get("firewall", rule, "enabled") == "0" + assert rule in disabled + assert u.get("firewall", zone, "enabled", default="XX") == "XX" # option must not be set + assert u.get("network", interface, "enabled", default="XX") == "XX" # option must not be set + for f in forwardings: + assert u.get("network", f, "enabled", default="XX") == "XX" # option must not be set + +def test_delete_linked_sections(u): + link = "mytestdb/mykey" + sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) + rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) + interface = firewall.add_vpn_interface(u, 'p2p', 'ppp10', link=link) + (zone, forwardings) = firewall.add_trusted_zone(u, 'mylinked3', link=link) + deleted = firewall.delete_linked_sections(u, link) + assert len(deleted) > 0 + with pytest.raises(UciExceptionNotFound): + u.get("firewall", rule) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", zone) + with pytest.raises(UciExceptionNotFound): + for s in sections: + u.get("firewall", s) + with pytest.raises(UciExceptionNotFound): + for f in forwardings: + u.get("firewall", f) + +def test_is_ipv6_enabled(u): + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'lan6') + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'wan6') + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'vlan6') + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'wan6b') + assert firewall.is_ipv6_enabled(u) == True + u.delete("network", 'wan6c') + assert firewall.is_ipv6_enabled(u) == False + +def test_disable_ipv6_firewall(u): + assert u.get("firewall", "v6rule", "enabled", default="1") == "1" + firewall.disable_ipv6_firewall(u) + assert u.get("firewall", "v6rule", "enabled", default="1") == "0" + + +def test_list_zones(u): + assert firewall.list_zones(u)["ns_lan"]["name"] == "lan" + assert firewall.list_zones(u)["ns_lan"]["input"] == "ACCEPT" + assert firewall.list_zones(u)["ns_lan"]["output"] == "ACCEPT" + assert firewall.list_zones(u)["ns_lan"]["forward"] == "ACCEPT" + assert firewall.list_zones(u)["ns_lan"]["network"] == ("GREEN_1", "al_lan") + assert firewall.list_zones(u)["ns_wan"]["name"] == "wan" + assert firewall.list_zones(u)["ns_wan"]["input"] == "REJECT" + assert firewall.list_zones(u)["ns_wan"]["output"] == "ACCEPT" + assert firewall.list_zones(u)["ns_wan"]["forward"] == "REJECT" + assert firewall.list_zones(u)["ns_wan"]["network"] == ("wan6", "RED_2", "RED_3", "RED_1") + assert firewall.list_zones(u)["ns_wan"]["log"] == "1" + + +def list_zones_no_aliases(u): + assert firewall.list_zones_no_aliases(u)["ns_lan"]["network"] == ("GREEN_1",) + + +def test_list_forwardings(u): + assert firewall.list_forwardings(u)["ns_lan2guests"]["src"] == "lan" + assert firewall.list_forwardings(u)["ns_lan2guests"]["dest"] == "guests" + assert firewall.list_forwardings(u)["ns_guests2wan"]["src"] == "guests" + assert firewall.list_forwardings(u)["ns_guests2wan"]["dest"] == "wan" + + +def test_add_zone(u): + assert firewall.add_zone(u, "new_zone", "REJECT", "DROP", True, ["lan"], ["lan", "guest"]) == ( + "ns_new_zone", {"ns_new_zone2wan", "ns_new_zone2lan", "ns_lan2new_zone", "ns_guest2new_zone"}) + assert u.get("firewall", "ns_new_zone", "name") == "new_zone" + assert u.get("firewall", "ns_new_zone", "input") == "REJECT" + assert u.get("firewall", "ns_new_zone", "output") == "ACCEPT" + assert u.get("firewall", "ns_new_zone", "forward") == "DROP" + assert u.get("firewall", "ns_new_zone", "log") == "0" + assert u.get("firewall", "ns_new_zone2wan", "src") == "new_zone" + assert u.get("firewall", "ns_new_zone2wan", "dest") == "wan" + assert u.get("firewall", "ns_new_zone2lan", "src") == "new_zone" + assert u.get("firewall", "ns_new_zone2lan", "dest") == "lan" + assert u.get("firewall", "ns_lan2new_zone", "src") == "lan" + assert u.get("firewall", "ns_lan2new_zone", "dest") == "new_zone" + assert u.get("firewall", "ns_guest2new_zone", "src") == "guest" + assert u.get("firewall", "ns_guest2new_zone", "dest") == "new_zone" + assert firewall.add_zone(u, "new_zone_with_log", "REJECT", "DROP", True, ["lan"], ["lan", "guest"], True) + assert u.get("firewall", "ns_new_zone_with_log", "log") == "1" + assert u.get("firewall", "ns_new_zone_with_log", "log_limit") == "5/s" + +def test_edit_zone(u): + assert firewall.edit_zone(u, "new_zone", "DROP", "ACCEPT", False, ["lan"], ["lan", "guest"]) == ( + "ns_new_zone", {"ns_new_zone2lan", "ns_lan2new_zone", "ns_guest2new_zone"}) + assert u.get("firewall", "ns_new_zone", "name") == "new_zone" + assert u.get("firewall", "ns_new_zone", "input") == "DROP" + assert u.get("firewall", "ns_new_zone", "output") == "ACCEPT" + assert u.get("firewall", "ns_new_zone", "forward") == "ACCEPT" + assert u.get("firewall", "ns_new_zone2lan", "src") == "new_zone" + assert u.get("firewall", "ns_new_zone2lan", "dest") == "lan" + assert u.get("firewall", "ns_lan2new_zone", "src") == "lan" + assert u.get("firewall", "ns_lan2new_zone", "dest") == "new_zone" + assert u.get("firewall", "ns_guest2new_zone", "src") == "guest" + assert u.get("firewall", "ns_guest2new_zone", "dest") == "new_zone" + assert u.get("firewall", "ns_new_zone", "log") == "0" + assert firewall.edit_zone(u, "new_zone", "DROP", "ACCEPT", False, ["lan"], ["lan", "guest"], True) + assert u.get("firewall", "ns_new_zone", "log") == "1" + + +def test_delete_zone(u): + assert firewall.delete_zone(u, "ns_new_zone") == ( + "ns_new_zone", {"ns_new_zone2lan", "ns_guest2new_zone", "ns_lan2new_zone"}) + with pytest.raises(Exception) as e: + firewall.delete_zone(u, "not_a_zone") + +def test_get_rule_by_name(u): + assert firewall.get_rule_by_name(u, "Allow-DHCPv6") == ( + "v6rule", + {"name": "Allow-DHCPv6", "src": "wan", "proto": "udp", "dest_port": "546", "family": "ipv6", "target": "ACCEPT", "enabled": "0"} + ) + assert firewall.get_rule_by_name(u, "Not-Automated") == ( + "manrule", + {"name": "Not-Automated", "src": "wan", "proto": "tcp", "dest_port": "1234", "family": "ipv4", "target": "ACCEPT"} + ) + assert firewall.get_rule_by_name(u, "not_a_rule") == (None, None) + assert firewall.get_rule_by_name(u, "Not-Automated", "automated") == (None, None) + +def test_add_default_ipv6_rules(u): + # one rule should be skipped because it already exists + assert len(firewall.add_default_ipv6_rules(u)) == 3 + assert firewall.add_default_ipv6_rules(u) == [] + +def test_resolve_address(u): + assert firewall.resolve_address(u, "192.168.100.1") == {"value": "192.168.100.1", "type": "domain", "label": "test.name.org"} + assert firewall.resolve_address(u, "192.168.100.2") == {"value": "192.168.100.2", "type": "host", "label": "test2.giacomo.org"} + assert firewall.resolve_address(u, "192.168.100.238") == {"value": "192.168.100.238", "type": "interface", "label": "lan"} + assert firewall.resolve_address(u, "ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543") == {"value": "ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543", "type": "domain", "label": "test3.test.org"} + assert firewall.resolve_address(u, "10.0.0.22") == {"value": "10.0.0.22", "type": "interface", "label": "bond1"} + assert firewall.resolve_address(u, "2001:db80::2/64") == {"value": "2001:db80::2/64", "type": "interface", "label": "wan6"} + +def test_is_zone(u): + assert firewall.is_zone(u,"lan") == True + assert firewall.is_zone(u,"wan") == True + assert firewall.is_zone(u,"blue") == True + assert firewall.is_zone(u,"wan") == True + # we expect it fails because the zone does not exist + assert firewall.is_zone(u,"lan2custom") == False + # we expect it to be true because we are using any zone + assert firewall.is_zone(u,"*") == True + +def test_list_forward_rules(u): + rules = firewall.list_forward_rules(u) + assert len(rules) > 0 + for r in rules: + # just check that only forwarded rules are returned + assert r.get("src") and r.get("dest") + # check all fields are presents + assert 'log' in r + assert 'enabled' in r + assert 'ns_tag' in r + assert 'proto' in r + assert 'active_zone' in r + +def test_list_output_rules(u): + rules = firewall.list_output_rules(u) + assert len(rules) > 0 + for r in rules: + # just check that only output rules are returned + assert r.get("src") is None and r.get("dest") + # check all fields are presents + assert 'log' in r + assert 'enabled' in r + assert 'ns_tag' in r + assert 'proto' in r + assert 'active_zone' in r + +def test_list_input_rules(u): + rules = firewall.list_input_rules(u) + assert len(rules) > 0 + for r in rules: + # just check that only input rules are returned + assert r.get("dest") is None and r.get("src") + # check all fields are presents + assert 'log' in r + assert 'enabled' in r + assert 'ns_tag' in r + assert 'proto' in r + assert 'active_zone' in r + +def test_list_service_suggestions(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + services = firewall.list_service_suggestions() + assert len(services) == 5 + assert services == [{'id': 'ftp', 'proto': ['tcp'], 'port': 21}, {'id': 'ssh', 'proto': ['tcp', 'udp'], 'port': 22}, {'id': 'time', 'proto': ['udp'], 'port': 37}, {'id': 'www', 'proto': ['tcp'], 'port': 80}, {'id': 'kerberos', 'proto': ['tcp', 'udp'], 'port': 88}] + +def test_list_host_suggestions(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=lease_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + suggestions = firewall.list_host_suggestions(u) + assert len(suggestions) == 10 + assert suggestions == [ + {'value': '192.168.100.1', 'label': 'test.name.org', 'type': 'domain'}, + {'value': '192.168.100.2', 'label': 'test2.giacomo.org', 'type': 'host'}, + {'value': 'ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543', 'label': 'test3.test.org', 'type': 'domain'}, + {'value': '7.8.9.1', 'label': 'host1', 'type': 'domain'}, + {'value': '192.168.100.238', 'label': 'lan', 'type': 'network'}, + {'value': '2001:db80::2/64', 'label': 'wan6', 'type': 'network'}, + {'value': '10.0.0.22', 'label': 'bond1', 'type': 'network'}, + {'value': '10.0.0.1/24', 'label': 'al_lan', 'type': 'network'}, + {'value': '192.168.1.228', 'label': 'test1', 'type': 'lease'}, + {'value': '192.168.1.219', 'label': 'test2', 'type': 'lease'}, + ] + +def test_add_rule(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + rid = firewall.add_rule(u, 'myrule', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "ssh", True, True, ['tag1'], False) + assert u.get("firewall", rid, "name") == "myrule" + assert u.get("firewall", rid, "src") == "lan" + assert u.get("firewall", rid, "dest") == "wan" + assert u.get_all("firewall", rid, "src_ip") == ("192.168.1.22",) + assert u.get_all("firewall", rid, "dest_ip") == ("1.2.3.4",) + assert u.get_all("firewall", rid, "proto") == ('tcp', 'udp') + assert u.get("firewall", rid, "dest_port") == "22" + assert u.get("firewall", rid, "target") == "ACCEPT" + assert u.get("firewall", rid, "ns_service") == "ssh" + assert u.get("firewall", rid, "enabled") == "1" + assert u.get("firewall", rid, "log") == "1" + assert u.get("firewall", rid, "log_limit") == "1/s" + assert u.get_all("firewall", rid, "ns_tag") == ("tag1",) + assert u.get("firewall", rid, "ns_link", default="notpresent") == "notpresent" + +def test_edit_rule(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + rid = firewall.add_rule(u, 'myrule2', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '22', 'ACCEPT', "ssh", True, True, ['tag1'], False) + with pytest.raises(ValidationError): + firewall.edit_rule(u, 'notpresent', 'myrule3', 'lan', [], 'wan', [], [], '', 'ACCEPT', None, True, True, ['tag1']) + rid2 = firewall.edit_rule(u, rid, 'myrule3', 'lan', [], 'blue', [], ['udp'], '22', 'DROP', "", False, False, ['tag2']) + assert rid == rid2 + assert u.get("firewall", rid, "name") == "myrule3" + assert u.get("firewall", rid, "src") == "lan" + assert u.get("firewall", rid, "dest") == "blue" + assert u.get_all("firewall", rid, "proto") == "all" + assert u.get("firewall", rid, "ns_service") == "*" + with pytest.raises(UciExceptionNotFound): + u.get_all("firewall", rid, "src_ip") + with pytest.raises(UciExceptionNotFound): + u.get_all("firewall", rid, "dest_ip") + with pytest.raises(UciExceptionNotFound): + u.get("firewall", rid, "dest_port") == "22" + assert u.get("firewall", rid, "target") == "DROP" + assert u.get("firewall", rid, "enabled") == "0" + assert u.get("firewall", rid, "log") == "0" + assert u.get_all("firewall", rid, "ns_tag") == ("tag2",) + rid3 = firewall.edit_rule(u, rid, 'myrule3', 'lan', [], 'blue', [], ['udp'], '22', 'DROP', "www", False, False, ['tag2']) + assert u.get("firewall", rid, "ns_service") == "www" + assert u.get_all("firewall", rid, "proto") == ('tcp',) + assert u.get("firewall", rid, "dest_port") == "80" + +def test_add_rule_with_notrack(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + rid = firewall.add_rule(u, 'notrack_rule', 'lan', ['192.168.1.0/24'], 'wan', [], [], '', 'NOTRACK', "*", True, False, [], False) + assert u.get("firewall", rid, "name") == "notrack_rule" + assert u.get("firewall", rid, "target") == "NOTRACK" + assert u.get("firewall", rid, "src") == "lan" + assert u.get("firewall", rid, "dest") == "wan" + assert u.get_all("firewall", rid, "src_ip") == ("192.168.1.0/24",) + assert u.get("firewall", rid, "enabled") == "1" + +def test_edit_rule_to_notrack(u, mocker): + mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) + mock_isfile = mocker.patch('os.path.isfile') + mock_isfile.return_value = True + rid = firewall.add_rule(u, 'rule_to_change', 'lan', [], 'wan', [], [], '', 'ACCEPT', "*", True, False, [], False) + assert u.get("firewall", rid, "target") == "ACCEPT" + firewall.edit_rule(u, rid, 'rule_to_change', 'lan', [], 'wan', [], [], '', 'NOTRACK', "*", True, False, []) + assert u.get("firewall", rid, "target") == "NOTRACK" + +def test_delete_rule(u): + ids = firewall.list_rule_ids(u) + id_to_delete = ids.pop() + firewall.delete_rule(u, id_to_delete) + assert id_to_delete not in firewall.list_rule_ids(u) + +def test_disable_rule(u): + ids = firewall.list_rule_ids(u) + id_to_disable = ids.pop() + firewall.disable_rule(u, id_to_disable) + assert u.get("firewall", id_to_disable, "enabled") == "0" + +def test_enable_rule(u): + ids = firewall.list_rule_ids(u) + id_to_enable = ids.pop() + firewall.disable_rule(u, id_to_enable) + assert u.get("firewall", id_to_enable, "enabled") == "0" + firewall.enable_rule(u, id_to_enable) + assert u.get("firewall", id_to_enable, "enabled") == "1" + +def test_order_rules(u, mocker): + # The firewall.order_rules function uses the uci binary to reorder the rules + # The test is usefull because uci behaves differently on a real machine + assert True + +def test_list_nat_rules(u): + names = [] + for r in firewall.list_nat_rules(u): + names.append(r.get("name")) + assert "source_NAT1_vpm" in names + assert "masquerade" in names + assert "cdn_via_router" in names + assert "SNAT_NSEC7_style" in names + assert "Allow-DHCPv6" not in names + +def test_add_nat_rule(u): + id1 = firewall.add_nat_rule(u, "myrule", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") + assert u.get("firewall", id1, "name") == "myrule" + assert u.get("firewall", id1, "target") == "SNAT" + assert u.get("firewall", id1, "src") == "lan" + assert u.get("firewall", id1, "src_ip") == "1.2.3.4" + assert u.get("firewall", id1, "dest_ip") == "6.7.8.9" + assert u.get_all("firewall", id1, "proto") == ("all",) + with pytest.raises(ValidationError): + firewall.add_nat_rule(u, "myrule", "REJECT") + id2 = firewall.add_nat_rule(u, "myrule3", "ACCEPT", dest_ip="1.2.3.4") + assert u.get("firewall", id2, "target") == "ACCEPT" + assert u.get("firewall", id2, "dest_ip") == "1.2.3.4" + assert u.get_all("firewall", id2, "proto") == ("all",) + assert u.get("firewall", id2, "src") == "*" + with pytest.raises(UciExceptionNotFound): + u.get("firewall", id2, "src_ip") + +def test_edit_nat_rule(u): + id = firewall.add_nat_rule(u, "myrule4", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") + firewall.edit_nat_rule(u, id, "myrule4b", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "3.3.3.3") + assert u.get("firewall", id, "name") == "myrule4b" + assert u.get("firewall", id, "snat_ip") == "3.3.3.3" + assert u.get("firewall", id, "src_ip") == "1.2.3.4" + assert u.get("firewall", id, "dest_ip") == "6.7.8.9" + assert u.get("firewall", id, "target") == "SNAT" + assert u.get_all("firewall", id, "proto") == ("all",) + assert u.get("firewall", id, "src") == "lan" + +def test_delete_nat_rule(u): + with pytest.raises(ValidationError): + firewall.delete_nat_rule(u, "notpresent") + id = firewall.add_nat_rule(u, "myrule5", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") + assert firewall.delete_nat_rule(u, id) == id + +def test_list_netmap_rules(u): + names = [] + for r in firewall.list_netmap_rules(u): + names.append(r.get("name")) + assert "source_nat1" in names + assert "dest_nat1" in names + +def test_add_netmap_rule(u): + with pytest.raises(ValidationError): + id1 = firewall.add_netmap_rule(u, "myrule", "10.50.51.0/24", "10.50.50.0/24", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") + id1 = firewall.add_netmap_rule(u, "myrule", "10.50.51.0/24", "", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") + assert u.get("netmap", id1, "name") == "myrule" + assert u.get("netmap", id1, "src") == "10.50.51.0/24" + assert u.get_all("netmap", id1, "device_in") == ("eth0",) + assert u.get_all("netmap", id1, "device_out") == ("eth1",) + assert u.get("netmap", id1, "map_from") == "10.10.10.0/24" + assert u.get("netmap", id1, "map_to") == "192.168.1.0/24" + id2 = firewall.add_netmap_rule(u, "myrule2", "", "10.50.50.0/24", ["eth0"], [], "10.10.10.0/24", "192.168.1.0/24") + assert u.get("netmap", id2, "name") == "myrule2" + with pytest.raises(UciExceptionNotFound): + u.get("netmap", id2, "src") + assert u.get("netmap", id2, "dest") == "10.50.50.0/24" + assert u.get_all("netmap", id2, "device_in") == ("eth0",) + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id2, "device_out") + assert u.get("netmap", id2, "map_from") == "10.10.10.0/24" + assert u.get("netmap", id2, "map_to") == "192.168.1.0/24" + +def test_edit_netmap_rule(u): + id = firewall.add_netmap_rule(u, "myrule3", "", "10.50.50.0/24", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") + firewall.edit_netmap_rule(u, id, "myrule3b", "", "10.50.51.0/24", [], [], "10.10.11.0/24", "192.168.2.0/24") + assert u.get("netmap", id, "name") == "myrule3b" + with pytest.raises(UciExceptionNotFound): + u.get("netmap", id, "src") + assert u.get("netmap", id, "dest") == "10.50.51.0/24" + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id, "device_out") + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id, "device_in") + assert u.get("netmap", id, "map_from") == "10.10.11.0/24" + assert u.get("netmap", id, "map_to") == "192.168.2.0/24" + +def test_delete_netmap_rule(u): + with pytest.raises(ValidationError): + firewall.delete_netmap_rule(u, "notpresent") + id = firewall.add_netmap_rule(u, "myrule3b", "", "10.50.51.0/24", None, None, "10.10.11.0/24", "192.168.2.0/24") + assert firewall.delete_netmap_rule(u, id) == id + +@patch("nethsec.utils.subprocess.run") +def test_list_netmap_devices(mock_run, u): + u.get_all("netmap", id, "device_out") + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id, "device_in") + assert u.get("netmap", id, "map_from") == "10.10.11.0/24" + assert u.get("netmap", id, "map_to") == "192.168.2.0/24" + +def test_delete_netmap_rule(u): + with pytest.raises(ValidationError): + firewall.delete_netmap_rule(u, "notpresent") + id = firewall.add_netmap_rule(u, "myrule3b", "", "10.50.51.0/24", None, None, "10.10.11.0/24", "192.168.2.0/24") + assert firewall.delete_netmap_rule(u, id) == id + +@patch("nethsec.utils.subprocess.run") +def test_list_netmap_devices(mock_run, u): + mock_run.return_value = mock_ip_stdout + devices = firewall.list_netmap_devices(u) + device_names = [d.get("device") for d in devices] + assert 'lo' not in device_names + assert 'br-lan' in device_names + assert 'eth1.4' in device_names + assert {'device': 'br-lan', 'interface': 'lan'} in devices + +def test_update_redirect_rules(u): + domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) + ipsets = objects.get_domain_set_ipsets(u, domain1) + host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) + u.set("firewall", "redirect3", "ns_src", f"objects/{domain1}") # domain can be used only as source + u.set('firewall', 'redirect3', 'ns_dst', f"dhcp/ns_dhcp1") + firewall.update_redirect_rules(u) + print(u.get_all("firewall", "redirect3")) + assert u.get("firewall", "redirect3", "dest_ip") == "7.8.9.1" + assert u.get("firewall", "redirect3", "ipset") == f"{ipsets['firewall']} src_net" + u.set("firewall", "redirect4", "ns_src", f"objects/{host1}") + u.set('firewall', 'redirect4', 'ns_dst', f"users/ns_user1") + firewall.update_redirect_rules(u) + assert u.get("firewall", "redirect4", "dest_ip") == "10.10.10.22" + assert u.get("firewall", "redirect4", "ipset") == f"redirect4_ipset" + assert u.get("firewall", "redirect4_ipset") + assert u.get('firewall', 'redirect4_ipset', 'ns_link') == f'firewall/redirect4' + +def test_update_firewall_rules(u): + domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) + ipsets = objects.get_domain_set_ipsets(u, domain1) + host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) + u.set("firewall", "r1", "ns_dst", f"objects/{domain1}") + u.set("firewall", "r1", "ns_src", f"objects/{host1}") + firewall.update_firewall_rules(u) + assert set(u.get_all("firewall", "r1", "src_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) + assert u.get("firewall", "r1", "ipset") == f"{ipsets['firewall']} dst" + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r1", "dest_ip") + + u.set("firewall", "r2", "ns_dst", f"objects/{host1}") + u.set("firewall", "r2", "ns_src", f"objects/{domain1}") + firewall.update_firewall_rules(u) + + assert u.get("firewall", "r2", "ipset") == f"{ipsets['firewall']} src" + assert set(u.get_all("firewall", "r2", "dest_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r2", "src_ip") + + u.set("firewall", "r3", "ns_src", "dhcp/ns_dhcp1") + u.set("firewall", "r3", "ns_dst", "users/ns_user1") + firewall.update_firewall_rules(u) + assert u.get_all("firewall", "r3", "src_ip") == ("7.8.9.1",) + assert u.get_all("firewall", "r3", "dest_ip") == ("10.10.10.22",) + + u.set("firewall", "r4", "ns_src", f"objects/{domain1}") + u.set("firewall", "r4", "dest_ip", "1.2.3.4") + firewall.update_firewall_rules(u) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r4", "ns_dst") + +def test_update_firewall_rules_concurrency(u): + host1 = objects.add_host_set(u, "hc1", "ipv4", ["1.1.1.1"]) + host2 = objects.add_host_set(u, "hc2", "ipv4", ["2.2.2.2"]) + rid = firewall.add_rule(u, 'mycrule', '*', [], 'wan', [], [], '', 'DROP', "", True, False, ['tag1'], False, f"objects/{host1}", f"objects/{host2}") + assert u.get("firewall", rid, "src_ip") == "1.1.1.1" + assert u.get("firewall", rid, "dest_ip") == "2.2.2.2" + firewall.update_firewall_rules(u) + firewall.update_redirect_rules(u) + assert u.get("firewall", rid, "src_ip") == "1.1.1.1" + assert u.get("firewall", rid, "dest_ip") == "2.2.2.2" + firewall.delete_rule(u, rid) + objects.delete_host_set(u, host1) + objects.delete_host_set(u, host2) + +def test_list_object_suggestions(u): + obj = objects.list_objects(u) + assert len(obj) == 9 + +def test_edit_rule_remove_object(u): + host1 = objects.add_host_set(u, "h1", "ipv4", ["1.2.3.4"]) + # def add_rule(uci, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled=True, log=False, tag=[], add_to_top=False, ns_src=None, ns_dst=None): + idf1 = firewall.add_rule(u, "forward1", "*", [], "wan", [], [], [], "REJECT", "*", True, False, [], False, f"objects/{host1}", "") + # remove object from rule + firewall.edit_rule(u, idf1, "forward1", "*", [], "wan", [], [], [], "REJECT", "*", True, False, []) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", idf1, "src_ip") + assert u.get('firewall', idf1, 'ns_src', default='NONE') == 'NONE' + firewall.delete_rule(u, idf1) + objects.delete_host_set(u, host1) + +def test_get_default_logging_options(u): + logging_options = firewall.get_default_logging_options(u) + assert logging_options['rule_log_limit'] == '1/s' + assert logging_options['zone_log_limit'] == '5/s' + assert logging_options['redirect_log_limit'] == '1/s' + +def test_rule_default_log_limit(u): + rid = firewall.add_rule(u, 'myrule3', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "", log=True) + assert u.get("firewall", rid, "log_limit") == "1/s" + +def test_change_rule_default_log_limit(u): + u.set('firewall', 'ns_defaults', 'rule_log_limit', '2/s') + u.save('firewall') + rid = firewall.add_rule(u, 'myrule4', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "", log=True) + assert u.get("firewall", rid, "log_limit") == "2/s" + +def test_assert_custom_log_limit(u): + # Check rule + rid = firewall.add_rule(u, 'myrule5', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "", log=True) + u.set('firewall', rid, 'log_limit', '123/s') + u.save('firewall') + rid = firewall.edit_rule(u, rid, 'myrule5', 'lan', ['192.168.1.22'], 'wan', ['1.2.3.4'], ['tcp', 'udp'], '443', 'ACCEPT', "", log=True) + assert u.get("firewall", rid, "log_limit") == "123/s" + # Check zone + (zid, forwardings) = firewall.add_zone(u, "ztwl", "REJECT", "DROP", log=True) + print(zid) + u.set('firewall', zid, 'log_limit', '123/s') + u.save('firewall') + firewall.edit_zone(u, zid, "DROP", "ACCEPT", log=True) + assert u.get('firewall', zid, 'log_limit') == '123/s' + +def test_change_zone_default_log_limit(u): + u.set("firewall", "ns_defaults", "zone_log_limit", "20/s") + u.save('firewall') + (zid, forwardings) = firewall.add_zone(u, "new_zone3", "REJECT", "DROP", True, ["lan"], ["lan", "guest"], True) + assert u.get("firewall", zid, "log_limit") == "20/s" + +def test_apply_default_logging_options(u): + firewall.apply_default_logging_options(u) + assert u.get("firewall", "ns_59eea167", "log_limit") == "1/s" + assert u.get("firewall", "f1", "log_limit") == "2/s" + assert u.get("firewall", "with_log", "log_limit") == "20/s" + assert u.get("firewall", "grey", "log_limit", default=None) == None + assert u.get("firewall", "o1", "log_limit", default=None) == None + assert u.get("firewall", "redirect3", "log_limit", default=None) == None + + +def test_ns_link_rules(u): + rid = firewall.add_rule(u, 'ns_link_rule', 'lan', [], 'wan', [], [], [], 'ACCEPT', '', ns_link='test/example') + assert u.get('firewall', rid, 'ns_link') == 'test/example' + firewall.edit_rule(u, rid, 'ns_link_rule', 'lan', [], 'wan', [], [], [], 'ACCEPT', '', ns_link='test/example2') + assert u.get('firewall', rid, 'ns_link') == 'test/example2' + firewall.edit_rule(u, rid, 'ns_link_rule', 'lan', [], 'wan', [], [], [], 'ACCEPT', '') + assert u.get('firewall', rid, 'ns_link', default=None) is None + rid = firewall.add_rule(u, 'ns_link_rule_no_link', 'lan', [], 'wan', [], [], [], 'ACCEPT', '') + assert u.get('firewall', rid, 'ns_link', default=None) is None \ No newline at end of file diff --git a/packages/python3-nethsec/tests/test_inventory.py b/packages/python3-nethsec/tests/test_inventory.py new file mode 100644 index 000000000..c81ab7eee --- /dev/null +++ b/packages/python3-nethsec/tests/test_inventory.py @@ -0,0 +1,1324 @@ +from euci import EUci +from unittest.mock import patch, mock_open, MagicMock +import subprocess + +from nethsec import inventory + +netmap_db = """ + +config rule 'ns_b606fef5' + option name 'foo' + option dest '8.8.8.0/24' + option map_from '12.0.0.0/24' + option map_to '16.0.0.0/24' + +config rule 'ns_003ff675' + option name 'john' + option src '12.0.0.0/24' + option map_from '15.0.0.0/24' + option map_to '14.0.0.0/24' +""" + +firewall_db = """ +config zone lan1 + option name 'lan' + list network 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list device 'eth0' + +config zone grey + option name 'grey' + list network 'grey1' + option input 'DROP' + option output 'ACCEPT' + option forward 'DROP' + list device 'eth1' + +config zone orange + option name 'orange' + list network 'orange1' + option input 'DROP' + option output 'DROP' + option forward 'DROP' + +config zone wan1f + option name 'wan' + list network 'wan' + list network 'wan6' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + +config forwarding fw1 + option src 'lan' + option dest 'wan' + +config rule 'v6rule' + option name 'Allow-DHCPv6' + option src 'wan' + option proto 'udp' + option dest_port '546' + option family 'ipv6' + option target 'ACCEPT' + +config rule 'manrule' + option name 'Not-Automated' + option src 'wan' + option proto 'tcp' + option dest_port '1234' + option family 'ipv4' + option target 'ACCEPT' + +config rule 'f1' + option name 'r1' + option dest 'wan' + option dest_port '22' + option target 'ACCEPT' + option src 'lan' + list src_ip '192.168.100.1' + list src_ip '192.168.100.238' + list dest_ip '192.168.122.1' + list dest_ip '192.168.122.49' + option log '1' + +config rule 'o1' + option name 'output1' + list dest_ip '192.168.100.1' + option target 'ACCEPT' + option dest 'wan' + +config rule 'i1' + option name 'Allow-OpenVPNRW1' + option src 'wan' + option dest_port '1194' + option proto 'udp' + option target 'ACCEPT' + list ns_tag 'automated' + option ns_link 'openvpn/ns_roadwarrior1' + +config nat + option name 'source_NAT1_vpm' + option src '*' + option src_ip '192.168.55.0/24' + option dest_ip '10.20.30.0/24' + option target 'SNAT' + option snat_ip '10.44.44.1' + list proto 'all' + +config nat + option name 'masquerade' + list proto 'all' + option src 'lan' + option src_ip '192.168.1.0/24' + option dest_ip '10.88.88.0/24' + option target 'MASQUERADE' + +config nat + option name 'cdn_via_router' + list proto 'all' + option src 'lan' + option src_ip '192.168.1.0/24' + option dest_ip '192.168.50.0/24' + option target 'ACCEPT' + +config nat + option name 'SNAT_NSEC7_style' + list proto 'all' + option src 'wan' + option src_ip '192.168.1.44' + option target 'SNAT' + option snat_ip '10.20.30.5' + +config redirect 'redirect3' + option ns_src '' + option ipset 'redirect3_ipset' + +config ipset 'redirect3_ipset' + option name 'redirect1_ipset' + option match 'src_net' + option enabled '1' + list entry '6.7.8.9' + +config redirect 'redirect4' + option ns_src '' + +config rule 'r1' + option name 'r1' + option ns_dst 'dhcp/ns_8dcab636' + +config rule 'r2' + option name 'r2' + option ns_dst '' + +config rule 'r3' + option name 'r3' + option ns_src '' + +config rule 'r4' + option ns_dst '' + option ns_src '' + +config rule 'ns_72004856' + option name 'forward' + option src '*' + option dest 'lan' + option target 'DROP' + option ns_service '*' + option proto 'all' + option enabled '1' + option log '0' + list src_ip '1.1.1.1' + option ns_dst 'objects/ns_7c9c0852' + option ipset 'foo dst' + +config rule 'ns_df2ca87c' + option name 'output' + option dest '*' + option target 'DROP' + option ns_service 'discard' + option dest_port '9' + option enabled '1' + option log '0' + list proto 'tcp' + list proto 'udp' + option ns_dst 'dhcp/ns_8d5d2cf4' + list dest_ip '192.168.50.123' + +config rule 'ns_a50bcca9' + option name 'input' + option src '*' + option target 'DROP' + option ns_service '*' + option proto 'all' + option enabled '1' + option log '0' + option ns_src 'objects/ns_bcfb447b' + list src_ip '192.168.50.123' + +""" + +network_db = """ +config interface 'lan' + option device 'br-lan' + option proto 'static' + option ipaddr '192.168.50.1' + option netmask '255.255.255.0' + +config interface 'wan' + option device 'eth1' + option proto 'dhcp' + +config interface 'wan6' + option device 'eth1' + option proto 'dhcpv6' + +config interface 'bond1' + option proto 'bonding' + option ipaddr '10.0.0.22' + option netmask '255.255.255.0' + list slaves 'eth3' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + option all_slaves_active '0' + option link_monitoring 'off' + +config interface 'bond2' + option proto 'bonding' + option ipaddr '10.11.11.22' + option netmask '255.255.255.0' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + list slaves 'eth3.2' + +config device 'ns_3b3556cf' + option name 'eth0.6' + option type '8021q' + option ifname 'eth0' + option vid '6' + option ipv6 '0' +config interface 'vlan6' + option device 'eth0.6' + option proto 'dhcp' + +config device 'ns_c3c5206a' + option name 'bond-foo' + option ipv6 '0' +config interface 'bond1' + option device 'bond-toto' + option proto 'static' + option ipaddr '125.12.12.12' + option netmask '255.255.255.0' +config route 'ns_31de6ad0' + option disabled '0' + option gateway '1.1.1.1' + option metric '0' + option mtu '1500' + option ns_description 'truc' + option onlink '0' + option target '1.1.1.0/24' + option type 'unicast' + +config route6 'ns_4a936870' + option disabled '0' + option gateway 'fe80::fc54:ff:fe70:2' + option metric '0' + option mtu '1500' + option ns_description 'peolopopo' + option onlink '0' + option target 'fe80::fc54:ff:fe70:d2bc/64' + option type 'unicast' + +""" + +dedalo_db = """ +config dedalo 'config' + option aaa_url 'https://my.nethspot.com/wax/aaa' + option api_url 'https://my.nethspot.com/api' + option network '192.168.82.0/24' + option disabled '0' + option splash_page 'http://my.nethspot.com/wings' + option max_clients '253' + option dhcp_end '250' + option dhcp_start '2' + option hotspot_id '51' + option unit_name 'fw1.nethsecurity.org' + option unit_description 'fw1 sede' + option interface 'eth2.10' + option secret 'xxxxxx' + option unit_uuid 'yyyyy' +""" + +flashstart_db = """ +config main 'global' + option enabled '0' + option proplus '1' + option password 'xxx' + option username 'xxx@nethsecurity.org' + list zones 'lan' + list bypass '192.168.1.252' + list bypass '192.168.1.211' + list custom_servers '/www.google.com/1.1.1.1' + """ + +openvpn_db = """ +config openvpn 'sample_client' + option client '1' + option dev 'tun' + option proto 'udp' + list remote 'my_server_1 1194' + option resolv_retry 'infinite' + option nobind '1' + option persist_key '1' + option persist_tun '1' + option user 'nobody' + option ca '/etc/openvpn/ca.crt' + option cert '/etc/openvpn/client.crt' + option key '/etc/openvpn/client.key' + option verb '3' + +config openvpn 'ns_openvpn1' + option dev 'tunopenvpn1' + option dev_type 'tun' + option enabled '1' + option persist_tun '1' + option float '1' + option multihome '1' + option passtos '1' + option ping_timer_rem '1' + option persist_key '1' + option keepalive '10 60' + option lport '1202' + option proto 'udp' + option topology 'subnet' + option dh '/etc/openvpn/ns_openvpn1/pki/dh.pem' + option ca '/etc/openvpn/ns_openvpn1/pki/ca.crt' + option cert '/etc/openvpn/ns_openvpn1/pki/issued/server.crt' + option key '/etc/openvpn/ns_openvpn1/pki/private/server.key' + option server '10.7.57.0 255.255.255.0' + list push 'topology subnet' + list push 'route 192.168.50.0 255.255.255.0' + list route '192.168.10.0 255.255.255.0' + option cipher 'AES-256-GCM' + option auth 'SHA256' + list ns_public_ip '86.206.129.103' + option ns_name 'openvpn1' + option client_connect '"/usr/libexec/ns-openvpn/openvpn-connect ns_openvpn1"' + option client_disconnect '"/usr/libexec/ns-openvpn/openvpn-disconnect ns_openvpn1"' + option management '/var/run/openvpn_ns_openvpn1.socket unix' + +config openvpn 'ns_roadwarrior1' + option proto 'udp' + option port '1194' + option dev 'tunrw1' + option dev_type 'tun' + option topology 'subnet' + option float '1' + option passtos '1' + option multihome '1' + option verb '3' + option enabled '1' + option keepalive '20 120' + option server '10.9.9.0 255.255.255.0' + option client_connect '"/usr/libexec/ns-openvpn/openvpn-connect ns_roadwarrior1"' + option client_disconnect '"/usr/libexec/ns-openvpn/openvpn-disconnect ns_roadwarrior1"' + option dh '/etc/openvpn/ns_roadwarrior1/pki/dh.pem' + option ca '/etc/openvpn/ns_roadwarrior1/pki/ca.crt' + option cert '/etc/openvpn/ns_roadwarrior1/pki/issued/server.crt' + option crl_verify '/etc/openvpn/ns_roadwarrior1/pki/crl.pem' + option key '/etc/openvpn/ns_roadwarrior1/pki/private/server.key' + option management '/var/run/openvpn_ns_roadwarrior1.socket unix' + option client_to_client '0' + option auth 'SHA256' + option cipher 'AES-256-GCM' + option tls_version_min '1.2' + option ns_auth_mode 'username_password_certificate' + list ns_tag 'automated' + option ns_user_db 'NethService' + option ns_description 'srv1' + option auth_user_pass_verify '/usr/libexec/ns-openvpn/openvpn-remote-auth via-env' + option script_security '3' + list push 'redirect-gateway' + list push 'route 192.168.1.0 255.255.255.0' + list push 'route 172.2.2.0 255.255.254.0' + list ns_public_ip 'vpn.test.org' + +config user 'user1' + option instance 'ns_roadwarrior1' + option ipaddr '10.9.9.50' + option enabled '1' +""" + +ns_plug_db = """ +config main 'config' + option tls_verify '1' + option backup_url 'https://backupd.nethsecurity.org/backup' + option type 'enterprise' + option alerts_url 'https://my.nethsecurity.org/isa/' + option api_url 'https://my.nethsecurity.org/api/' + option inventory_url 'https://my.nethsecurity.org/isa/inventory/store/' + option system_id 'xxx' + option secret 'yyy' + option server '' + option unit_id '' + option token '' +""" + +ban_ip_db = """ +config banip 'global' + option ban_enabled '1' + option ban_debug '1' + option ban_autodetect '0' + list ban_logterm 'Exit before auth from' + option ban_fetchcmd 'curl' + option ban_protov4 '1' + list ban_ifv4 'FIBER100M' + list ban_ifv4 'FTTC_BCK' + list ban_dev 'eth0' + list ban_dev 'eth5' + list ban_allowurl '' + option ban_loginput '1' + option ban_logforwardwan '1' + option ban_logforwardlan '1' + option ban_autoallowlist '1' + option ban_autoblocklist '1' + option ban_allowlistonly '0' + option ban_nftexpiry '1d' + option ban_logcount '3' + list ban_feed 'nethesislvl3' + list ban_feed 'test1' +""" + +ns_ui_db = """ +config main 'config' + option luci_enable '0' + option nsui_enable '1' + option nsui_extra_port '9090' + option nsui_extra_enable '1' +""" + +fstab_db = """ +config global + option anon_swap '0' + option anon_mount '0' + option auto_swap '1' + option auto_mount '1' + option delay_root '5' + option check_fs '0' + +config mount + option target '/rom' + option uuid '520aae54-7d368ea4-c0a96e03-92927a0f' + option enabled '0' + +config mount 'ns_data' + option target '/mnt/data' + option uuid '523ba406-b2b6-4a3c-b071-ac1bf6acf4c6' + option enabled '1' +""" + +nginx_db = """ +config main 'global' + option uci_enable 'true' + +config server '_lan' + list listen '443 ssl default_server' + list listen '[::]:443 ssl default_server' + option server_name '_lan' + list include 'conf.d/*.locations' + option uci_manage_ssl 'custom' + option ssl_certificate '/etc/ssl/acme/test.nethsecurity.org.fullchain.crt' + option ssl_certificate_key '/etc/ssl/acme/test.nethsecurity.org.key' + option ssl_session_cache 'shared:SSL:32k' + option ssl_session_timeout '64m' + option error_log 'syslog:server=unix:/dev/log' + option access_log 'syslog:server=unix:/dev/log' + +config location 'ns_server2_location1' + option uci_server 'ns_server2' + option location '/' + option proxy_pass 'https://172.4.5.4' + +config server 'ns_server1' + option uci_description 'Test 1' + option ssl_session_timeout '64m' + list proxy_set_header 'Host $http_host' + list listen '443 ssl' + list listen '[::]:443 ssl' + option ssl_session_cache 'shared:SSL:32k' + option server_name 'metrics.nethsecurity.org' + list include 'conf.d/ns_server1.proxy' + option uci_manage_ssl 'acme' + option ssl_certificate '/etc/ssl/acme/test.nethsecurity.org.fullchain.crt' + option ssl_certificate_key '/etc/ssl/acme/test.nethsecurity.org.key' + +config location 'ns_server1_location1' + option uci_server 'ns_server1' + option location '/' + option proxy_pass 'https://172.3.3.3' +""" + +ipsec_db = """ +config ipsec 'ns_ipsec_global' + option debug '1' + option zone 'ipsec' + +config crypto_proposal 'ns_6fd94f07_ike' + option encryption_algorithm 'aes256' + option hash_algorithm 'sha256' + option dh_group 'modp2048' + option ns_link 'ipsec/ns_6fd94f07' + +config crypto_proposal 'ns_6fd94f07_esp' + option encryption_algorithm 'aes256' + option hash_algorithm 'sha256' + option dh_group 'modp2048' + option ns_link 'ipsec/ns_6fd94f07' + +config tunnel 'ns_6fd94f07_tunnel_1' + option ipcomp 'false' + option dpdaction 'none' + list local_subnet '192.168.1.0/24' + list remote_subnet '192.168.3.0/24' + option rekeytime '3600' + list crypto_proposal 'ns_6fd94f07_esp' + option closeaction 'none' + option startaction 'start' + option if_id '1' + option ns_link 'ipsec/ns_6fd94f07' + +config tunnel 'ns_6fd94f07_tunnel_2' + option ipcomp 'false' + option dpdaction 'none' + list local_subnet '172.4.4.0/24' + list remote_subnet '192.168.3.0/24' + option rekeytime '3600' + list crypto_proposal 'ns_6fd94f07_esp' + option closeaction 'none' + option startaction 'start' + option if_id '1' + option ns_link 'ipsec/ns_6fd94f07' + +config remote 'ns_6fd94f07' + option ns_name 't1' + option authentication_method 'psk' + option gateway '1.2.3.4' + option keyexchange 'ikev1' + option local_identifier '@l1' + option local_ip '5.6.7.8' + option enabled '1' + option remote_identifier '@r1' + option pre_shared_key 'xxx' + list crypto_proposal 'ns_6fd94f07_ike' + option rekeytime '3600' + list tunnel 'ns_6fd94f07_tunnel_1' + list tunnel 'ns_6fd94f07_tunnel_2' +""" + +dpi_db = """ +config main 'config' + option log_blocked '0' + option enabled '1' + option firewall_exemption '0' + +config rule 'ns_2b170d05' + option enabled '1' + option device 'br-lan' + option action 'block' + list application 'netify.facebook' +""" + +dhcp_db = """ +config dnsmasq + option domainneeded '1' + option boguspriv '1' + option filterwin2k '0' + option localise_queries '1' + option rebind_protection '1' + option rebind_localhost '1' + option local '/lan/' + option domain 'nethesis.it' + option expandhosts '1' + option nonegcache '0' + option cachesize '1000' + option authoritative '1' + option readethers '1' + option leasefile '/tmp/dhcp.leases' + option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto' + option nonwildcard '1' + option localservice '1' + option ednspacket_max '1232' + option filter_aaaa '0' + option filter_a '0' + option logqueries '0' + list server '8.8.8.8' + list server '8.8.4.4' + list rebind_domain 'nethsecurity.org' + +config dhcp 'lan' + option interface 'lan' + option start '100' + option limit '150' + option leasetime '12h' + option dhcpv4 'server' + option dhcpv6 'server' + option ra 'server' + option ra_slaac '1' + list ra_flags 'managed-config' + list ra_flags 'other-config' +""" + +mwan3_db = """ +config globals 'globals' + option mmx_mask '0x3F00' + +config policy 'ns_default' + option label 'Default' + list use_member 'ns_FIBER100M_M10_W100' + list use_member 'ns_FTTC_BCK_M20_W100' + +config interface 'FIBER100M' + option enabled '1' + option initial_state 'online' + option family 'ipv4' + list track_ip '8.8.8.8' + list track_ip '208.67.222.222' + option track_method 'ping' + option reliability '1' + option count '1' + option size '56' + option max_ttl '60' + option timeout '4' + option interval '10' + option failure_interval '5' + option recovery_interval '5' + option down '5' + option up '5' + +config member 'ns_FIBER100M_M10_W100' + option interface 'FIBER100M' + option metric '10' + option weight '100' + +config interface 'FTTC_BCK' + option enabled '1' + option initial_state 'online' + option family 'ipv4' + list track_ip '8.8.8.8' + list track_ip '208.67.222.222' + option track_method 'ping' + option reliability '1' + option count '1' + option size '56' + option max_ttl '60' + option timeout '4' + option interval '10' + option failure_interval '5' + option recovery_interval '5' + option down '5' + option up '5' + +config member 'ns_FTTC_BCK_M20_W100' + option interface 'FTTC_BCK' + option metric '20' + option weight '100' + +config rule 'ns_default_rule' + option label 'Default Rule' + option use_policy 'ns_default' + option sticky '1' +""" + +qosify_db = """ +config interface 'wan' + option name 'wan' + option disabled '1' + option bandwidth_up '100mbit' + option bandwidth_down '100mbit' + option overhead_type 'none' + option ingress '1' + option egress '1' + option mode 'diffserv4' + option nat '1' + option host_isolate '1' + option autorate_ingress '0' + +config device 'wandev' + option disabled '1' + option name 'wan' + option bandwidth '100mbit' + +config interface 'FIBER100M' + option name 'FIBER100M' + option disabled '0' + option bandwidth_up '85mbit' + option bandwidth_down '85mbit' + +config interface 'FTTC_BCK' + option name 'FTTC_BCK' + option disabled '0' + option bandwidth_up '20mbit' + option bandwidth_down '40mbit' +""" + +objects_db = """ + +config rule 'ns_b606fef5' + option name 'object1' + option dest '8.8.8.0/24' + option map_from '12.0.0.0/24' + option map_to '16.0.0.0/24' + +config rule 'ns_003ff675' + option name 'object2' + option src '12.0.0.0/24' + option map_from '15.0.0.0/24' + option map_to '14.0.0.0/24' + +config domain 'ns_7c9c0852' + option name 'object3' + option family 'ipv4' + option timeout '660' + list domain 'foo.com' + +config host 'ns_bcfb447b' + option name 'object4' + option family 'ipv4' + list ipaddr 'dhcp/ns_8d5d2cf4' +""" + +adblock_db = """ +config adblock 'global' + option adb_enabled '1' + option adb_debug '0' + option adb_forcedns '1' + option adb_safesearch '0' + option adb_dnsfilereset '0' + option adb_mail '0' + option adb_report '0' + option adb_backup '1' + option adb_fetchutil 'wget' + option adb_dns 'dnsmasq' + option ts_enabled '1' + list adb_zonelist 'lan' + list adb_portlist '53' + list adb_portlist '853' + option adb_srcarc '/etc/adblock/combined.sources.gz' + option adb_dnsinstance '0' + option adb_fetchparm '--compression=gzip --no-cache --no-cookies --max-redirect=0 --timeout=20 -O' + list adb_sources 'adaway' + list adb_sources 'adguard' + list adb_sources 'disconnect' + list adb_sources 'yoyo' + list adb_sources 'malware_lvl2' + list adb_sources 'yoroi_susp_level2' + list adb_sources 'yoroi_malware_level1' +""" + +rpcd_db = """ +config rpcd + option dummy 'to get ignored' + +config login + option username 'root' + +config login 'controller' + option username 'gibblerish' + +config login 'ns_7fe3e016' + option username 'example' +""" + +def _setup_db(tmp_path): + # setup fake db + with tmp_path.joinpath('network').open('w') as fp: + fp.write(network_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('dedalo').open('w') as fp: + fp.write(dedalo_db) + with tmp_path.joinpath('flashstart').open('w') as fp: + fp.write(flashstart_db) + with tmp_path.joinpath('openvpn').open('w') as fp: + fp.write(openvpn_db) + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write(ns_plug_db) + with tmp_path.joinpath('banip').open('w') as fp: + fp.write(ban_ip_db) + with tmp_path.joinpath('ns-ui').open('w') as fp: + fp.write(ns_ui_db) + with tmp_path.joinpath('fstab').open('w') as fp: + fp.write(fstab_db) + with tmp_path.joinpath('nginx').open('w') as fp: + fp.write(nginx_db) + with tmp_path.joinpath('ipsec').open('w') as fp: + fp.write(ipsec_db) + with tmp_path.joinpath('dpi').open('w') as fp: + fp.write(dpi_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('mwan3').open('w') as fp: + fp.write(mwan3_db) + with tmp_path.joinpath('qosify').open('w') as fp: + fp.write(qosify_db) + with tmp_path.joinpath('netmap').open('w') as fp: + fp.write(netmap_db) + with tmp_path.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with tmp_path.joinpath('adblock').open('w') as fp: + fp.write(adblock_db) + with tmp_path.joinpath('rpcd').open('w') as fp: + fp.write(rpcd_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_fact_hotspot(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_hotspot(u) == {"enabled": True, "server": "https://my.nethspot.com/api", "interface": "eth2.10"} + +def test_fact_flashstart(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_flashstart(u) == {"enabled": False, "bypass": 2, "custom_servers": 1, "pro_plus": True} + +def test_fact_openvpn_rw(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_openvpn_rw(u) == { + "enabled": 1, + "server": 1, + "instances": [ + { + "section": "ns_roadwarrior1", + "authentication": "username_password_certificate", + "user_database": "NethService", + "mode": "tun" + } + ] + } + +def test_fact_openvpn_tun(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_openvpn_tun(u) + assert result == { + "client": 0, + "server": 1, + "tunnels": [ + { + "section": "ns_openvpn1", + "mode": "tun" + } + ] + } + +def test_fact_subscription_status(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_subscription_status(u) == {"status": "enterprise"} + +def test_fact_controller(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_controller(u) == {"enabled": False} + +def test_fact_rpcd_users(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_rpcd_users(u) == {"count": 1} + +def test_fact_threat_shield(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_threat_shield(u) == {"enabled": True, "community": 1, "enterprise": 1} + +def test_fact_ui(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_ui(u) == {"luci": False, "port443": True, "port9090": True} + +def test_fact_storage(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_storage(u) == {"enabled": True} + +def test_fact_network(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_network(u) + assert result['zones'][0]['name'] == 'lan' + assert result['zones'][0]['ipv4'] == 1 + assert result['zones'][0]['ipv6'] == 0 + assert result['zones'][1]['name'] == 'grey' + assert result['zones'][1]['ipv4'] == 1 + assert result['zones'][1]['ipv6'] == 0 + assert result['zones'][2]['name'] == 'orange' + assert result['zones'][2]['ipv4'] == 0 + assert result['zones'][2]['ipv6'] == 0 + assert result['zones'][3]['name'] == 'wan' + assert result['zones'][3]['ipv4'] == 1 + assert result['zones'][3]['ipv6'] == 0 + assert result['zones'][4]['name'] == 'mytrusted' + assert result['zones'][4]['ipv4'] == 0 + assert result['zones'][4]['ipv6'] == 0 + assert result['zones'][5]['name'] == 'mylinked' + assert result['zones'][5]['ipv4'] == 0 + assert result['zones'][5]['ipv6'] == 0 + assert result['zones'][6]['name'] == 'mytrusted2' + assert result['zones'][6]['ipv4'] == 0 + assert result['zones'][6]['ipv6'] == 0 + assert result['zones'][7]['name'] == 'blue' + assert result['zones'][7]['ipv4'] == 1 + assert result['zones'][7]['ipv6'] == 0 + assert result['interface_counts']['vlans'] == 0 + assert result['interface_counts']['bridges'] == 2 + assert result['interface_counts']['bonds'] == 0 + assert result['zone_network_counts']['lan'] == 2 + assert result['zone_network_counts']['grey'] == 1 + assert result['zone_network_counts']['orange'] == 0 + assert result['zone_network_counts']['wan'] == 2 + assert result['zone_network_counts']['mytrusted'] == 0 + assert result['zone_network_counts']['mylinked'] == 0 + assert result['zone_network_counts']['mytrusted2'] == 0 + assert result['zone_network_counts']['blue'] == 1 + assert result['route_info']['count_ipv6_route'] == 1 + assert result['route_info']['count_ipv4_route'] == 1 + +def test_fact_proxy_pass(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_proxy_pass(u) == {"count": 2} + +def test_fact_ipsec(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_ipsec(u) == {"count": 1} + +def test_fact_dpi(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_dpi(u) == {"enabled": True, "rules": 1} + +def test_fact_dhcp_server(tmp_path): + u = _setup_db(tmp_path) + assert inventory.fact_dhcp_server(u) == {"count": 1, "static_leases": 0, "dynamic_leases": 0, "dns_records_count": 0, "dns_forwarder_enabled": True } + +def test_fact_multiwan(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_multiwan(u) + assert result['enabled'] + assert result['policies']['backup'] == 1 + assert result['policies']['balance'] == 0 + assert result['policies']['custom'] == 0 + assert result['rules'] == 1 + +def test_fact_qos(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_qos(u) + assert result['count'] == 2 + assert result['rules'][0]['enabled'] == True + assert result['rules'][0]['upload'] == 85 + assert result['rules'][0]['download'] == 85 + assert result['rules'][1]['enabled'] == True + assert result['rules'][1]['upload'] == 20 + assert result['rules'][1]['download'] == 40 + +def test_fact_certificates_info(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_certificates_info(u) + assert result['custom_certificates']['count'] == 0 + assert result['acme_certificates']['count'] == 0 + assert result['acme_certificates']['issued'] == 0 + assert result['acme_certificates']['pending'] == 0 + +def test_fact_firewall_stats(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_firewall_stats(u) + + # Validate the 'firewall' section + assert result['firewall']['port_forward'] == 2 + assert result['firewall']['nat']['masquerade'] == 1 + assert result['firewall']['nat']['snat'] == 4 + assert result['firewall']['nat']['accept'] == 2 + assert result['firewall']['netmap']['source'] == 3 + assert result['firewall']['netmap']['destination'] == 2 + assert result['firewall']['rules']['forward'] == 19 + assert result['firewall']['rules']['input'] == 7 + assert result['firewall']['rules']['output'] == 2 + + # Validate the 'objects' section + assert result['objects']['domains'] == 3 + assert result['objects']['hosts'] == 3 + assert result['objects']['port_forward']['allowed_from'] == 2 + assert result['objects']['port_forward']['destination_to'] == 2 + assert result['objects']['mwan_rules'] == 0 + assert result['objects']['rules']['forward'] == 1 + assert result['objects']['rules']['input'] == 1 + assert result['objects']['rules']['output'] == 1 + +def test_fact_adblock(tmp_path): + u = _setup_db(tmp_path) + result = inventory.fact_adblock(u) + assert result == {"enabled": True, "community": 5, "enterprise": 2} + +# Tests for info_* helper functions + +def test_info_kernel_version(): + """Test info_kernel_version reads from /proc/version""" + u = EUci() + + # Test successful read + mock_version = "Linux version 5.10.176 (builder@nethsec) (gcc version 11.2.0) #0 SMP Mon Jan 1 00:00:00 2024\n" + with patch('builtins.open', mock_open(read_data=mock_version)): + result = inventory.info_kernel_version(u) + assert result == "5.10.176" + + # Test exception handling (file not found) + with patch('builtins.open', side_effect=FileNotFoundError): + result = inventory.info_kernel_version(u) + assert result == '' + +def test_info_uptime_seconds(): + """Test info_uptime_seconds reads from /proc/uptime""" + u = EUci() + + # Test successful read + mock_uptime = "12345.67 98765.43\n" + with patch('builtins.open', mock_open(read_data=mock_uptime)): + result = inventory.info_uptime_seconds(u) + assert result == 12345 + + # Test exception handling + with patch('builtins.open', side_effect=FileNotFoundError): + result = inventory.info_uptime_seconds(u) + assert result == 0 + +def test_info_fqdn(tmp_path): + """Test info_fqdn retrieves hostname from UCI""" + # Setup system config + system_db = """ +config system + option hostname 'myhost' + option domain 'example.com' +""" + with tmp_path.joinpath('system').open('w') as fp: + fp.write(system_db) + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + + u = EUci(confdir=tmp_path.as_posix()) + result = inventory.info_fqdn(u) + # Should be anonymized since no subscription + assert result.startswith('anon-') + +def test_info_default_ipv4_dig_method(tmp_path): + """Test info_default_ipv4 with dig method""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock successful dig command + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '"192.0.2.1"\n' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_default_ipv4(u) + assert result.startswith('anon-') # Should be anonymized + +def test_info_default_ipv4_curl_fallback(tmp_path): + """Test info_default_ipv4 falls back to curl when dig fails""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock dig failure and curl success + def mock_subprocess_run(cmd, **kwargs): + result = MagicMock() + if 'dig' in cmd: + result.returncode = 1 + result.stdout = '' + elif 'curl' in cmd: + result.returncode = 0 + result.stdout = '203.0.113.1\n' + else: + result.returncode = 1 + result.stdout = '' + return result + + with patch('subprocess.run', side_effect=mock_subprocess_run): + result = inventory.info_default_ipv4(u) + assert result.startswith('anon-') + +def test_info_default_ipv4_ip_command_fallback(tmp_path): + """Test info_default_ipv4 falls back to ip command when dig and curl fail""" + with tmp_path.joinpath('network').open('w') as fp: + fp.write(network_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock dig and curl failure, ip command success + def mock_subprocess_run(cmd, **kwargs): + result = MagicMock() + if isinstance(cmd, list) and 'ip' in cmd: + result.returncode = 0 + result.stdout = '[{"addr_info": [{"family": "inet", "local": "198.51.100.1"}]}]' + else: + result.returncode = 1 + result.stdout = '' + return result + + with patch('subprocess.run', side_effect=mock_subprocess_run): + with patch('nethsec.utils.get_all_wan_devices', return_value=['eth0']): + result = inventory.info_default_ipv4(u) + assert result.startswith('anon-') + +def test_info_default_ipv4_all_methods_fail(tmp_path): + """Test info_default_ipv4 returns empty string when all methods fail""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock all methods failing + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stdout = '' + + with patch('subprocess.run', return_value=mock_result): + with patch('nethsec.utils.get_all_wan_devices', return_value=[]): + result = inventory.info_default_ipv4(u) + assert result == '' + +def test_info_default_ipv6_no_wan_ipv6(tmp_path): + """Test info_default_ipv6 returns empty when no IPv6 on WAN""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock no WAN devices or no IPv6 + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '[]' + + with patch('subprocess.run', return_value=mock_result): + with patch('nethsec.utils.get_all_wan_devices', return_value=['eth0']): + result = inventory.info_default_ipv6(u) + assert result == '' + +def test_info_default_ipv6_with_wan_ipv6(tmp_path): + """Test info_default_ipv6 with IPv6 on WAN""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock WAN with IPv6 + def mock_subprocess_run(cmd, **kwargs): + result = MagicMock() + if isinstance(cmd, list) and 'ip' in cmd and '-6' in cmd: + result.returncode = 0 + result.stdout = '[{"addr_info": [{"family": "inet6", "local": "2001:db8::1", "scope": "global"}]}]' + elif isinstance(cmd, list) and 'dig' in cmd: + result.returncode = 0 + result.stdout = '"2001:db8::2"\n' + else: + result.returncode = 1 + result.stdout = '' + return result + + with patch('subprocess.run', side_effect=mock_subprocess_run): + with patch('nethsec.utils.get_all_wan_devices', return_value=['eth0']): + result = inventory.info_default_ipv6(u) + assert result.startswith('anon-') + +def test_info_default_ipv6_fallback_to_curl(tmp_path): + """Test info_default_ipv6 falls back to curl when dig fails""" + with tmp_path.joinpath('ns-plug').open('w') as fp: + fp.write("config config\n\toption type 'no'\n") + u = EUci(confdir=tmp_path.as_posix()) + + # Mock WAN with IPv6, dig fails, curl succeeds + def mock_subprocess_run(cmd, **kwargs): + result = MagicMock() + if isinstance(cmd, list) and 'ip' in cmd and '-6' in cmd: + result.returncode = 0 + result.stdout = '[{"addr_info": [{"family": "inet6", "local": "2001:db8::1", "scope": "global"}]}]' + elif isinstance(cmd, list) and 'curl' in cmd: + result.returncode = 0 + result.stdout = '2001:db8::3\n' + else: + result.returncode = 1 + result.stdout = '' + return result + + with patch('subprocess.run', side_effect=mock_subprocess_run): + with patch('nethsec.utils.get_all_wan_devices', return_value=['eth0']): + result = inventory.info_default_ipv6(u) + assert result.startswith('anon-') + +def test_info_package_updates_available_with_updates(): + """Test info_package_updates_available when updates are available""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"updates": ["package1", "package2"]}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_package_updates_available(u) + assert result is True + +def test_info_package_updates_available_no_updates(): + """Test info_package_updates_available when no updates available""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"updates": []}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_package_updates_available(u) + assert result is False + +def test_info_package_updates_available_command_fails(): + """Test info_package_updates_available when command fails""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stdout = '' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_package_updates_available(u) + assert result is False + +def test_parse_version_with_prefix(): + """Test parse_version removes NethSecurity prefix""" + assert inventory.parse_version('NethSecurity 8.1.0-dev') == (8, 1, 0) + assert inventory.parse_version('NethSecurity 8.2.1') == (8, 2, 1) + +def test_parse_version_without_prefix(): + """Test parse_version handles versions without prefix""" + assert inventory.parse_version('8.1.0-dev') == (8, 1, 0) + assert inventory.parse_version('8.2.1') == (8, 2, 1) + +def test_parse_version_comparison(): + """Test parse_version version comparison logic""" + v1 = inventory.parse_version('8.1.0') + v2 = inventory.parse_version('8.2.0') + v3 = inventory.parse_version('8.1.1') + + assert v2 > v1 + assert v3 > v1 + assert v2 > v3 + +def test_parse_version_invalid(): + """Test parse_version handles invalid input""" + result = inventory.parse_version('invalid') + assert result == () + + result = inventory.parse_version('') + assert result == () + +def test_info_image_updates_available_update_exists(): + """Test info_image_updates_available when newer version exists""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.1.0", "lastVersion": "NethSecurity 8.2.0"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is True + +def test_info_image_updates_available_no_update(): + """Test info_image_updates_available when current version is latest""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.2.0", "lastVersion": "NethSecurity 8.2.0"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is False + +def test_info_image_updates_available_patch_update(): + """Test info_image_updates_available when a patch update is available""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.2.0", "lastVersion": "NethSecurity 8.2.1"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is True + +def test_info_image_updates_available_minor_update(): + """Test info_image_updates_available when a minor update is available""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.2.0", "lastVersion": "NethSecurity 8.3.0"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is True + +def test_info_image_updates_available_current_newer(): + """Test info_image_updates_available when current version is newer than last""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"currentVersion": "NethSecurity 8.3.0", "lastVersion": "NethSecurity 8.2.0"}' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is False + +def test_info_image_updates_available_command_fails(): + """Test info_image_updates_available when command fails""" + u = EUci() + + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stdout = '' + + with patch('subprocess.run', return_value=mock_result): + result = inventory.info_image_updates_available(u) + assert result is False diff --git a/packages/python3-nethsec/tests/test_ipsec.py b/packages/python3-nethsec/tests/test_ipsec.py new file mode 100644 index 000000000..ba1b38ef4 --- /dev/null +++ b/packages/python3-nethsec/tests/test_ipsec.py @@ -0,0 +1,89 @@ +import pathlib + +import pytest +from euci import EUci +from pytest_mock import MockFixture +from nethsec import ipsec, utils + +templates_db = """ +config template_rule 'ns_ipsec_esp' + option name 'Allow-IPSec-ESP' + option src 'wan' + option proto 'esp' + option target 'ACCEPT' + +config template_rule 'ns_ipsec_ike' + option name 'Allow-IPSec-IKE' + option src 'wan' + option dest_port '500' + option proto 'udp' + option target 'ACCEPT' + +config template_rule 'ns_ipsec_nat' + option name 'Allow-IPSec-NAT' + option src 'wan' + option dest_port '500' + option proto 'udp' + option target 'ACCEPT' +""" + +@pytest.fixture +def e_uci(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + return EUci(confdir=conf_dir.as_posix(), savedir=save_dir.as_posix()) + + +@pytest.fixture +def e_uci_with_data(e_uci: EUci): + with pathlib.Path(e_uci.confdir()).joinpath('templates').open('w') as fp: + fp.write(templates_db) + with pathlib.Path(e_uci.confdir()).joinpath('firewall').open('a') as fp: + pass + return e_uci + +def test_init_ipsec(e_uci): + ipsec.init_ipsec(e_uci) + assert(e_uci.get('ipsec', 'ns_ipsec_global') == 'ipsec') + assert(e_uci.get('ipsec', 'ns_ipsec_global', 'debug') == '0') + assert(e_uci.get('ipsec', 'ns_ipsec_global', 'zone') == ipsec.IPSEC_ZONE) + +def test_open_firewall_ports(e_uci_with_data): + ipsec.open_firewall_ports(e_uci_with_data) + nat = ike = esp = False + for r in utils.get_all_by_type(e_uci_with_data, 'firewall', 'rule'): + name = e_uci_with_data.get('firewall', r, 'name') + if name == 'Allow-IPSec-NAT': + nat = True + elif name == 'Allow-IPSec-IKE': + ike = True + elif name == 'Allow-IPSec-ESP': + esp = True + assert (nat and ipsec and esp) + +def test_add_trusted_interface(e_uci_with_data): + ipsec.add_trusted_interface(e_uci_with_data, 'ipsec1') + count = 0 + zid = '' + # check the zone has been created + for section in e_uci_with_data.get_all('firewall'): + if e_uci_with_data.get('firewall', section) == 'zone': + if e_uci_with_data.get('firewall', section, 'name') == ipsec.IPSEC_ZONE: + count = count + 1 + zid = section + assert(count == 1) + assert(zid) + assert(e_uci_with_data.get_all('firewall', zid, 'network') == ('ipsec1',)) + # check the zone has not been duplicated + count = 0 + ipsec.add_trusted_interface(e_uci_with_data, 'ipsec2') + for section in e_uci_with_data.get_all('firewall'): + if e_uci_with_data.get('firewall', section) == 'zone': + if e_uci_with_data.get('firewall', section, 'name') == ipsec.IPSEC_ZONE: + count = count + 1 + assert(count == 1) + assert('ipsec1' in e_uci_with_data.get_all('firewall', zid, 'network')) + assert('ipsec2' in e_uci_with_data.get_all('firewall', zid, 'network')) + diff --git a/packages/python3-nethsec/tests/test_mwan.py b/packages/python3-nethsec/tests/test_mwan.py new file mode 100644 index 000000000..5b6a9ce33 --- /dev/null +++ b/packages/python3-nethsec/tests/test_mwan.py @@ -0,0 +1,610 @@ +import pathlib + +import pytest +from euci import EUci + +from nethsec import mwan, objects +from nethsec.utils import ValidationError + +network_db = """ +config interface 'loopback' + option device 'lo' + option proto 'static' + option ipaddr '127.0.0.1' + option netmask '255.0.0.0' + +config interface 'GREEN_1' + option proto 'static' + option device 'eth0' + option ipaddr '192.168.200.2' + option netmask '255.255.255.0' + +config interface 'RED_1' + option proto 'static' + option device 'eth1' + option ipaddr '10.0.0.2' + option netmask '255.255.255.0' + option gateway '10.0.0.1' + +config interface 'RED_2' + option proto 'static' + option device 'eth2' + option ipaddr '10.0.1.2' + option netmask '255.255.255.0' + option gateway '10.0.1.1' + +config interface 'RED_3' + option proto 'static' + option device 'eth3' + option ipaddr '10.0.2.2' + option netmask '255.255.255.0' + option gateway '10.0.2.1' + +config device + option name 'eth0' + +config device + option name 'eth1' + +config device + option name 'eth2' + +config device + option name 'eth3' +""" + +ns_api_db = """ +config defaults_mwan 'defaults_mwan' + option initial_state 'online' + option protocol 'ipv4' + list track_ip '8.8.8.8' + list track_ip '208.67.222.222' + option tracking_method 'ping' + option tracking_reliability '1' + option ping_count '1' + option ping_size '56' + option ping_max_ttl '60' + option ping_timeout '4' + option ping_interval '10' + option ping_failure_interval '5' + option ping_recovery_interval '5' + option interface_down_threshold '5' + option interface_up_threshold '5' + option link_quality '0' + option quality_failure_latency '1000' + option quality_failure_packet_loss '40' + option quality_recovery_latency '500' + option quality_recovery_packet_loss '10' +""" + +objects_db = """ +""" + +dhcp_db = """ +config domain 'ns_domain_mwan' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' + +config host 'ns_host_mwan' + option ip '192.168.100.5' + option mac 'fe:54:00:6a:50:bf' + option dns '1' + option name 'host2' + option ns_description 'host2' +""" + +users_db = """ +""" + +firewall_db = """ +""" + +@pytest.fixture +def e_uci(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + with conf_dir.joinpath('network').open('w') as fp: + fp.write(network_db) + with conf_dir.joinpath('ns-api').open('w') as fp: + fp.write(ns_api_db) + with conf_dir.joinpath('mwan3').open('w') as fp: + fp.write('') + with conf_dir.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with conf_dir.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with conf_dir.joinpath('users').open('w') as fp: + fp.write(users_db) + with conf_dir.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + return EUci(confdir=conf_dir.as_posix(), savedir=save_dir.as_posix()) + + +def test_create_interface(e_uci): + assert mwan.__store_interface(e_uci, 'RED_1') == (True, True) + assert mwan.__store_interface(e_uci, 'RED_2') == (True, True) + # assert every interface has defaults + assert e_uci.get('mwan3', 'RED_1') == 'interface' + assert e_uci.get('mwan3', 'RED_1', 'enabled') == '1' + assert e_uci.get('mwan3', 'RED_1', 'initial_state') == 'online' + assert e_uci.get('mwan3', 'RED_1', 'family') == 'ipv4' + assert e_uci.get('mwan3', 'RED_1', 'track_ip', list=True) == ('8.8.8.8', '208.67.222.222') + assert e_uci.get('mwan3', 'RED_1', 'track_method') == 'ping' + assert e_uci.get('mwan3', 'RED_1', 'reliability') == '1' + assert e_uci.get('mwan3', 'RED_1', 'count') == '1' + assert e_uci.get('mwan3', 'RED_1', 'size') == '56' + assert e_uci.get('mwan3', 'RED_1', 'max_ttl') == '60' + assert e_uci.get('mwan3', 'RED_1', 'timeout') == '4' + assert e_uci.get('mwan3', 'RED_1', 'interval') == '10' + assert e_uci.get('mwan3', 'RED_1', 'failure_interval') == '5' + assert e_uci.get('mwan3', 'RED_1', 'recovery_interval') == '5' + assert e_uci.get('mwan3', 'RED_1', 'down') == '5' + assert e_uci.get('mwan3', 'RED_1', 'up') == '5' + # assert interface has metric + assert e_uci.get('network', 'RED_1', 'metric') == '30' + assert e_uci.get('network', 'RED_2', 'metric') == '40' + assert mwan.__store_interface(e_uci, 'RED_1') == (False, False) + + +def test_fail_create_invalid_interface(e_uci): + with pytest.raises(ValueError) as err: + mwan.__store_interface(e_uci, 'RED_4') + assert err.value.args[0] == 'name' + assert err.value.args[1] == 'invalid' + assert err.value.args[2] == 'RED_4' + + +def test_interface_avoid_edit_of_metric(e_uci): + e_uci.set('network', 'RED_1', 'metric', '10') + assert mwan.__store_interface(e_uci, 'RED_1') == (True, False) + + +def test_create_member(e_uci): + assert mwan.__store_member(e_uci, 'RED_1', 10, 100) == ('ns_RED_1_M10_W100', True) + assert mwan.__store_member(e_uci, 'RED_1', 10, 100) == ('ns_RED_1_M10_W100', False) + assert mwan.__store_member(e_uci, 'RED_1', 1, 100) == ('ns_RED_1_M1_W100', True) + + +def test_create_default_mwan(e_uci, mocker): + mocker.patch('subprocess.run') + assert mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '20', + 'weight': '100', + } + ]) == ['mwan3.ns_default', + 'mwan3.RED_1', + 'network.RED_1', + 'mwan3.ns_RED_1_M10_W200', + 'mwan3.RED_2', + 'network.RED_2', + 'mwan3.ns_RED_2_M20_W100', + 'mwan3.ns_default_rule'] + + assert e_uci.get('mwan3', 'ns_default') == 'policy' + assert e_uci.get('mwan3', 'ns_default', 'label') == 'default' + assert e_uci.get('mwan3', 'ns_default', 'use_member', list=True) == ( + 'ns_RED_1_M10_W200', 'ns_RED_2_M20_W100') + assert e_uci.get('mwan3', 'ns_default', 'last_resort') == 'default' + + +def test_create_unique_mwan(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'this', []) + with pytest.raises(ValueError): + mwan.store_policy(e_uci, 'this', []) + + +def test_metric_generation(e_uci): + assert mwan.__generate_metric(e_uci) == 30 + assert mwan.__store_interface(e_uci, 'RED_1') == (True, True) + assert mwan.__generate_metric(e_uci) == 40 + assert mwan.__generate_metric(e_uci) == 40 + assert mwan.__store_interface(e_uci, 'RED_2') == (True, True) + assert mwan.__generate_metric(e_uci) == 50 + + +def test_list_policies(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'backup', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '20', + 'weight': '100', + } + ]) + mwan.store_policy(e_uci, 'balance', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + mwan.store_policy(e_uci, 'custom', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + index = mwan.index_policies(e_uci) + # check backup policy + assert index[0]['name'] == 'ns_backup' + assert index[0]['label'] == 'backup' + assert index[0]['type'] == 'backup' + assert index[0]['members'][10][0]['name'] == 'ns_RED_1_M10_W200' + assert index[0]['members'][10][0]['interface'] == 'RED_1' + assert index[0]['members'][10][0]['metric'] == '10' + assert index[0]['members'][10][0]['weight'] == '200' + assert index[0]['members'][20][0]['name'] == 'ns_RED_2_M20_W100' + assert index[0]['members'][20][0]['interface'] == 'RED_2' + assert index[0]['members'][20][0]['metric'] == '20' + assert index[0]['members'][20][0]['weight'] == '100' + # check balance policy + assert index[1]['name'] == 'ns_balance' + assert index[1]['label'] == 'balance' + assert index[1]['type'] == 'balance' + assert index[1]['members'][10][0]['name'] == 'ns_RED_3_M10_W200' + assert index[1]['members'][10][0]['interface'] == 'RED_3' + assert index[1]['members'][10][0]['metric'] == '10' + assert index[1]['members'][10][0]['weight'] == '200' + assert index[1]['members'][10][1]['name'] == 'ns_RED_2_M10_W100' + assert index[1]['members'][10][1]['interface'] == 'RED_2' + assert index[1]['members'][10][1]['metric'] == '10' + assert index[1]['members'][10][1]['weight'] == '100' + # check custom policy + assert index[2]['name'] == 'ns_custom' + assert index[2]['label'] == 'custom' + assert index[2]['type'] == 'custom' + assert index[2]['members'][10][0]['name'] == 'ns_RED_3_M10_W200' + assert index[2]['members'][10][0]['interface'] == 'RED_3' + assert index[2]['members'][10][0]['metric'] == '10' + assert index[2]['members'][10][0]['weight'] == '200' + assert index[2]['members'][10][1]['name'] == 'ns_RED_2_M10_W100' + assert index[2]['members'][10][1]['interface'] == 'RED_2' + assert index[2]['members'][10][1]['metric'] == '10' + assert index[2]['members'][10][1]['weight'] == '100' + assert index[2]['members'][20][0]['name'] == 'ns_RED_1_M20_W100' + assert index[2]['members'][20][0]['interface'] == 'RED_1' + assert index[2]['members'][20][0]['metric'] == '20' + assert index[2]['members'][20][0]['weight'] == '100' + +def test_policy_length(e_uci, mocker): + mocker.patch('subprocess.run') + with pytest.raises(ValidationError) as e: + assert mwan.store_policy(e_uci, 'nameisa15maxlength', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '200', + }, + { + 'name': 'RED_2', + 'metric': '20', + 'weight': '100', + } + ]) + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'length_12_max' + +def test_store_rule(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + assert mwan.store_rule(e_uci, 'rule 1', 'ns_default', 'udp', '192.168.1.1/24', '1-1024', '10.0.0.2/12', + '22,443', True) == 'mwan3.ns_rule_1' + assert e_uci.get('mwan3', 'ns_rule_1') == 'rule' + assert e_uci.get('mwan3', 'ns_rule_1', 'label') == 'rule 1' + assert e_uci.get('mwan3', 'ns_rule_1', 'use_policy') == 'ns_default' + assert e_uci.get('mwan3', 'ns_rule_1', 'proto') == 'udp' + assert e_uci.get('mwan3', 'ns_rule_1', 'src_ip') == '192.168.1.1/24' + assert e_uci.get('mwan3', 'ns_rule_1', 'src_port') == '1:1024' + assert e_uci.get('mwan3', 'ns_rule_1', 'dest_ip') == '10.0.0.2/12' + assert e_uci.get('mwan3', 'ns_rule_1', 'dest_port') == '22,443' + assert e_uci.get('mwan3', 'ns_rule_1', 'sticky') == '1' + + domain_id = objects.add_domain_set(e_uci, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + with pytest.raises(ValueError): + mwan.store_rule(e_uci, 'r_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst=f"objects/{domain_id}") + + hostset_id = objects.add_host_set(e_uci, "myhostset", "ipv4", ["192.168.1.1"]) + id = mwan.store_rule(e_uci, 'r_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst=f"objects/{hostset_id}") + + id = id.split('.')[1] + assert e_uci.get('mwan3', id, 'ns_src') == "dhcp/ns_host_mwan" + assert e_uci.get('mwan3', id, 'ns_dst') == f"objects/{hostset_id}" + with pytest.raises(ValueError): + mwan.store_rule(e_uci, 'rule_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst="objects/invalid") + with pytest.raises(ValueError): + mwan.store_rule(e_uci, 'rule_with_obj', 'ns_default', 'udp', ns_src=f"objects/{domain_id}") + +def test_unique_rule(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + with pytest.raises(ValidationError) as e: + mwan.store_rule(e_uci, 'rule 1', 'ns_default') + mwan.store_rule(e_uci, 'rule 1', 'ns_default') + + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'unique' + +def test_rule_length(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + with pytest.raises(ValidationError) as e: + mwan.store_rule(e_uci, 'nameisa15maxlength', 'ns_default') + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'length_12_max' + + +def test_missing_policy_rule(e_uci): + with pytest.raises(ValidationError) as e: + mwan.store_rule(e_uci, 'cool rule', 'ns_default') + + assert e.value.args[0] == 'policy' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'ns_default' + + +def test_delete_non_existent_policy(e_uci, mocker): + mocker.patch('subprocess.run') + with pytest.raises(ValidationError) as e: + mwan.delete_policy(e_uci, 'ns_default') + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'ns_default' + + +def test_delete_policy(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + } + ]) + assert mwan.delete_policy(e_uci, 'ns_default') == ['mwan3.ns_default'] + assert e_uci.get('mwan3', 'ns_default', default=None) is None + + +def test_edit_policy(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + assert mwan.index_policies(e_uci)[0]['type'] == 'balance' + assert mwan.edit_policy(e_uci, 'ns_default', 'new label', [ + { + 'name': 'RED_1', + 'metric': '20', + 'weight': '100', + }, + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '100', + } + ]) == ['mwan3.ns_default', 'mwan3.ns_RED_1_M20_W100', 'mwan3.RED_3', 'network.RED_3', 'mwan3.ns_RED_3_M10_W100'] + assert e_uci.get('mwan3', 'ns_default', 'label') == 'new label' + assert mwan.index_policies(e_uci)[0]['type'] == 'backup' + + +def test_missing_policy(e_uci, mocker): + mocker.patch('subprocess.run') + with pytest.raises(ValidationError) as e: + mwan.edit_policy(e_uci, 'dummy', '', []) + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'dummy' + + +def test_index_rules(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + mwan.store_rule(e_uci, 'rule 1', 'ns_default') + index = mwan.index_rules(e_uci) + assert index[0] == { + 'name': 'ns_default_rule', + 'label': 'Default Rule', + 'policy': { + 'name': 'ns_default', + 'label': 'default', + }, + "sticky": False, + } + assert index[1] == { + 'name': 'ns_rule_1', + 'label': 'rule 1', + 'policy': { + 'name': 'ns_default', + 'label': 'default', + }, + "sticky": False, + } + + +def test_delete_rule(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'default', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + mwan.store_rule(e_uci, 'rule 1', 'ns_default') + assert mwan.delete_rule(e_uci, 'ns_rule_1') == 'mwan3.ns_rule_1' + assert 'ns_rule_1' not in e_uci.get_all('mwan3').keys() + + +def test_edit_rule(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'hello world', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + mwan.store_policy(e_uci, 'cool policy', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + } + ]) + assert mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label!', 'udp', '192.168.10.1/12', '80,443', + '0.0.0.0/0', '4040-8080') == 'mwan3.ns_default_rule' + assert e_uci.get('mwan3', 'ns_default_rule', 'label') == 'new label!' + assert e_uci.get('mwan3', 'ns_default_rule', 'use_policy') == 'ns_cool_policy' + assert e_uci.get('mwan3', 'ns_default_rule', 'proto') == 'udp' + assert e_uci.get('mwan3', 'ns_default_rule', 'src_ip') == '192.168.10.1/12' + assert e_uci.get('mwan3', 'ns_default_rule', 'src_port') == '80,443' + assert e_uci.get('mwan3', 'ns_default_rule', 'dest_ip') == '0.0.0.0/0' + assert e_uci.get('mwan3', 'ns_default_rule', 'dest_port') == '4040:8080' + + assert mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label2!', ns_src="dhcp/ns_host_mwan") + assert e_uci.get('mwan3', 'ns_default_rule', 'ns_src') == "dhcp/ns_host_mwan" + + +def test_cant_edit_invalid_rule(e_uci, mocker): + mocker.patch('subprocess.run') + with pytest.raises(ValidationError) as e: + mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label!') + assert e.value.args[0] == 'name' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'ns_default_rule' + mwan.store_policy(e_uci, 'hello world', [ + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_2', + 'metric': '10', + 'weight': '100', + } + ]) + with pytest.raises(ValidationError) as e: + mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label!') + assert e.value.args[0] == 'policy' + assert e.value.args[1] == 'invalid' + assert e.value.args[2] == 'ns_cool_policy' + +def test_policy_type(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'custom', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '200', + } + ]) + assert mwan.index_policies(e_uci)[0]['type'] == 'custom' + +def test_update_rules(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'cool policy', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + } + ]) + domain_id = objects.add_domain_set(e_uci, "mydomainset7", "ipv4", ["test1.com", "test2.com"]) + with pytest.raises(ValidationError): + mwan.store_rule(e_uci, 'r_with_obj', 'ns_cool_policy', 'udp', ns_src="dhcp/ns_domain_mwan", ns_dst=f"objects/{domain_id}") + + id = mwan.store_rule(e_uci, 'r_with_obj', 'ns_cool_policy', 'udp', ns_src="dhcp/ns_domain_mwan", ns_dst=f"dhcp/ns_host_mwan") + id = id.split('.')[1] + mwan.update_rules(e_uci) + assert e_uci.get('mwan3', id, 'src_ip') == '7.8.9.1' + assert e_uci.get('mwan3', id, 'dest_ip') == '192.168.100.5' diff --git a/packages/python3-nethsec/tests/test_objects.py b/packages/python3-nethsec/tests/test_objects.py new file mode 100644 index 000000000..4ef6a4a79 --- /dev/null +++ b/packages/python3-nethsec/tests/test_objects.py @@ -0,0 +1,380 @@ +import pathlib +from nethsec.utils import ValidationError +import pytest +from euci import EUci, UciExceptionNotFound +from nethsec import firewall, objects + +objects_db = """ +config domain 'myset' + option name 'MySet' + option description 'Mydomain set' + option family 'ipv4' + option timeout '600' + list domain 'www.nethsecurity.org' + list domain 'www.nethserver.org' + +config host 'ns_04fadb5c' + option name 'h1' + option family 'ipv4' + list ipaddr '1.2.3.4' + +config host 'ns_12345' + option name 'h1' + option family 'ipv4' + list ipaddr 'dhcp/ns_112233' + +config host 'ns_a7439990' + option name 'h1v6' + option family 'ipv6' + list ipaddr 'fd:618c:d80a:dc82:2380:54c7:7a17:1013' +""" + +firewall_db = """ +config rule 'r5' + option name 'r5' + option ns_dst 'dhcp/ns_8dcab636' + +config rule 'r6' + option name 'r6' + +config rule 'r7' + option name 'r7' + option ns_src 'objects/ns_04fadb5c' + +config rule 'r8' + option name 'r8' + +config redirect 'redirect1' + option ns_src '' + option ipset 'redirect1_ipset' + +config ipset 'redirect1_ipset' + option name 'redirect1_ipset' + option match 'src_net' + option enabled '1' + list entry '6.7.8.9' + +config redirect 'redirect2' + option ns_src '' +""" + +dhcp_db = """ +config domain 'ns_8bec5896' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' + +config domain 'ns_112233' + option ip '7.8.9.1' + option name 'used_host' + option ns_description 'Used Host' + +config host 'ns_8dcab636' + option ip '192.168.100.5' + option mac 'fe:54:00:6a:50:bf' + option dns '1' + option name 'host2' + option ns_description 'host2' + +config domain 'ns_9e7f705e' + option ip '1.2.3.4' + option name 'test1.domain' + +config host 'ns_271ca281' + option ip '192.168.100.22' + option mac 'fe:54:00:6a:4a:a1' + option dns '1' + option name 'reserve1' + option ns_description 'reserve1' +""" + +user_db = """ +config user 'ns_user1' + option name "john" + option database "main" + option label "John Doe" + option openvpn_ipaddr "10.10.10.22" + +config user 'ns_user2' + option name "user2" +""" + +mwan3_db = """ +config rule 'ns_r1' + option label 'r1' + option use_policy 'ns_default' + option sticky '0' + option proto 'tcp' + option src_ip '1.2.3.4' + option dest_ip '1.1.1.1' + option ns_src 'users/ns_user2' +""" + +dpi_db = """ +config rule + option action 'block' + list application 'netify.twitter' + list source 'dhcp/ns_9e7f705e' + option description 'Block Twitter for user Goofy' + option enabled '1' + +config exemption + option criteria 'dhcp/ns_271ca281' + option description 'Important host' + option enabled '1' + +config rule 'ns_e775b8a7' + option enabled '1' + option device 'br-lan' + option action 'block' + list application 'netify.amazon-prime' + +""" + +@pytest.fixture +def u(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() + with tmp_path.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('users').open('w') as fp: + fp.write(user_db) + with tmp_path.joinpath('mwan3').open('w') as fp: + fp.write(mwan3_db) + with tmp_path.joinpath('dpi').open('w') as fp: + fp.write(dpi_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_add_doman_set(u): + id1 = objects.add_domain_set(u, "mydomainset", "ipv4", ["test1.com", "test2.com"]) + assert u.get("objects", id1, "name") == "mydomainset" + assert u.get("objects", id1, "family") == "ipv4" + assert u.get_all("objects", id1, "domain") == ("test1.com", "test2.com") + assert u.get("objects", id1, "timeout") == "660" + + linked = firewall.get_all_linked(u, f"objects/{id1}") + assert linked['firewall'] is not None + assert u.get('firewall', linked['firewall'][0], 'ns_link') == f"objects/{id1}" + assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset" + assert u.get('firewall', linked['firewall'][0], 'family') == 'ipv4' + assert u.get('firewall', linked['firewall'][0], 'timeout') == '660' + + assert linked['dhcp'] is not None + assert u.get('dhcp', linked['dhcp'][0], 'ns_link') == f"objects/{id1}" + assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset",) + assert u.get_all('dhcp', linked['dhcp'][0], 'domain') == ("test1.com", "test2.com") + + id2 = objects.add_domain_set(u, "mydomainset2", "ipv6", ["test3.com", "test4.com"], 300) + assert u.get("objects", id2, "name") == "mydomainset2" + assert u.get("objects", id2, "family") == "ipv6" + assert u.get_all("objects", id2, "domain") == ("test3.com", "test4.com") + assert u.get("objects", id2, "timeout") == "300" + linked = firewall.get_all_linked(u, f"objects/{id2}") + assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset2" + assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset2",) + +def test_edit_domain_set(u): + id = objects.add_domain_set(u, "mydomainset3", "ipv4", ["test1.com", "test2.com"]) + objects.edit_domain_set(u, id, "mydomainset3b", "ipv6", ["test3.com", "test4.com"], 600) + assert u.get("objects", id, "name") == "mydomainset3b" + assert u.get("objects", id, "family") == "ipv6" + assert u.get_all("objects", id, "domain") == ("test3.com", "test4.com") + assert u.get("objects", id, "timeout") == "600" + +def test_delete_domain_set(u): + with pytest.raises(ValidationError): + objects.delete_domain_set(u, "notpresent") + id = objects.add_domain_set(u, "mydomainset4", "ipv4", ["test1.com", "test2.com"]) + assert objects.delete_domain_set(u, id) == id + linked = firewall.get_all_linked(u, f"objects/{id}") + assert linked['firewall'] == [] + assert linked['dhcp'] == [] + id = objects.add_domain_set(u, "mydomainsettd", "ipv4", ["test1.com", "test2.com"]) + u.set('firewall', 'r8', 'ns_dst', f"objects/{id}") + with pytest.raises(ValidationError): + objects.delete_domain_set(u, id) + +def test_is_used_domain_set(u): + id = objects.add_domain_set(u, "used1", "ipv4", ["test1.com", "test2.com"]) + u.set('firewall', 'r5', 'ns_dst', f"objects/{id}") + used, matches = objects.is_used_domain_set(u, id) + assert used + assert matches == ["firewall/r5"] + +def test_list_domain_sets(u): + sets = objects.list_domain_sets(u) + assert len(sets) == 8 + +def test_add_host_set(u): + with pytest.raises(ValidationError): + objects.add_host_set(u, "myhostset", "ipv4", ["a.b.c.d", "e.f.g.h"]) + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "4.5.6.0/24", "192.168.1.3-192.168.1.10"]) + assert u.get("objects", id1, "name") == "myhostset" + assert u.get_all("objects", id1, "ipaddr") == ("1.2.3.4", "4.5.6.0/24", "192.168.1.3-192.168.1.10") + assert u.get("objects", id1, "family") == "ipv4" + id2 = objects.add_host_set(u, "myhostset2", "ipv6", ["2001:db8:3333:4444:5555:6666:7777:8888", "2001:db8::/95", "2001:db8:3333:4444:5555:6666:7777:8888-2001:db8:3333:4444:5555:6666:7777:8890"]) + assert u.get("objects", id2, "name") == "myhostset2" + assert u.get("objects", id2, "family") == "ipv6" + assert u.get_all("objects", id2, "ipaddr") == ("2001:db8:3333:4444:5555:6666:7777:8888", "2001:db8::/95", "2001:db8:3333:4444:5555:6666:7777:8888-2001:db8:3333:4444:5555:6666:7777:8890") + +def test_edit_host_set(u): + id = objects.add_host_set(u, "myhostset3", "ipv4", ["6.7.8.9"]) + objects.edit_host_set(u, id, "myhostset3b", "ipv4", ["1.1.1.1", "2.2.2.2"]) + assert u.get("objects", id, "name") == "myhostset3b" + assert u.get_all("objects", id, "ipaddr") == ("1.1.1.1", "2.2.2.2") + +def test_delete_host_set(u): + with pytest.raises(ValidationError): + objects.delete_host_set(u, "notpresent") + id = objects.add_host_set(u, "myhostset4", "ipv4", ["6.7.8.9"]) + assert objects.delete_host_set(u, id) == id + id = objects.add_host_set(u, "myhostsettd", "ipv4", ["2.2.2.2"]) + u.set('firewall', 'r8', 'ns_dst', f"objects/{id}") + with pytest.raises(ValidationError): + objects.delete_host_set(u, id) + +def test_is_used_host_set(u): + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.1.1.1"]) + u.set('firewall', 'r6', 'ns_src', f"objects/{id}") + used, matches = objects.is_used_host_set(u, id) + assert used + assert matches == ["firewall/r6"] + +def test_list_host_sets(u): + sets = objects.list_host_sets(u) + assert len(sets) == 10 + +def test_is_singleton_host_set(u): + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "5.6.7.8"]) + assert objects.is_singleton_host_set(u, f"objects/{id1}") == False + id2 = objects.add_host_set(u, "myhostset2", "ipv4", ["8.8.8.8"]) + assert objects.is_singleton_host_set(u, f"objects/{id2}") == True + id3 = objects.add_host_set(u, "myhostset3", "ipv4", ["192.168.7.2-192.168.7.3"]) + assert objects.is_singleton_host_set(u, f"objects/{id3}") == False + id4 = objects.add_host_set(u, "myhostset4", "ipv4", ["192.168.0.0/24"]) + assert objects.is_singleton_host_set(u, f"objects/{id4}") == False + assert objects.is_singleton_host_set(u, f"objects/{id4}", allow_cidr=True) + +def test_add_host_set_loopback(u): + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + id2 = objects.add_host_set(u, "myhostset2", "ipv4", [f"objects/{id1}"]) + with pytest.raises(ValidationError): + objects.edit_host_set(u, id1, "myhostset", "ipv4", [f"objects/{id2}"]) + +def test_is_used_object(u): + used, matches = objects.is_used_object(u, "dhcp/ns_8dcab636") + assert used + assert matches == ["firewall/r5"] + assert objects.is_used_object(u, "dhcp/ns_8bec5896")[0] == False + assert objects.is_used_object(u, "users/ns_user2")[0] == True + used, matches = objects.is_used_object(u, "objects/ns_04fadb5c") + assert used + assert matches == ["firewall/r7"] + used, matches = objects.is_used_object(u, "dhcp/ns_112233") + assert used + assert matches == ["objects/ns_12345"] + +def test_object_exists(u): + assert objects.object_exists(u, "users/ns_user1") + assert objects.object_exists(u, "uknown") == False + +def test_get_object(u): + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + obj = objects.get_object(u, id) + +def test_get_object_ips(u): + id0 = objects.add_host_set(u, "myhostset0", "ipv4", ["4.5.6.7"]) + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "dhcp/ns_8bec5896", "users/ns_user1", f"objects/{id0}"]) + ips = objects.get_object_ips(u, f"objects/{id}") + assert set(ips) == set(["1.2.3.4", "7.8.9.1", "10.10.10.22", "4.5.6.7"]) # check with set to ignore order + +def test_is_domain_set(u): + id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + assert objects.is_domain_set(u, f"objects/{id}") == True + assert objects.is_domain_set(u, "dhcp/ns_8dcab636") == False + assert objects.is_domain_set(u, "users/ns_user1") == False + +def test_is_domain(u): + id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + assert objects.is_domain(u, f"objects/{id}") == False + assert objects.is_domain(u, "dhcp/ns_8bec5896") + +def test_is_host(u): + assert objects.is_host(u, "dhcp/ns_8bec5896") == False + assert objects.is_host(u, "dhcp/ns_8dcab636") + +def test_is_vpn_user(u): + assert objects.is_vpn_user(u, "users/ns_user1") + assert objects.is_vpn_user(u, "users/ns_user2") == False + +def test_is_host_set(u): + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + assert objects.is_host_set(u, f"objects/{id}") + assert objects.is_host_set(u, "dhcp/ns_8dcab636") == False + +def test_list_vpn_users(u): + users = objects.list_vpn_users(u) + assert len(users) == 1 + users = objects.list_vpn_users(u, True) + assert 'ipaddr' in users[0] + +def test_list_dns_records(u): + records = objects.list_dns_records(u) + assert len(records) == 3 + records = objects.list_dns_records(u, True) + for r in records: + assert 'ipaddr' in r + +def test_list_dhcp_static_leases(u): + leases = objects.list_dhcp_static_leases(u) + assert len(leases) == 2 + leases = objects.list_dhcp_static_leases(u, True) + for l in leases: + assert 'ipaddr' in l + +def test_list_objects(u): + objs = objects.list_objects(u) + assert len(objs) == len(objects.list_domain_sets(u)) + len(objects.list_host_sets(u)) + len(objects.list_vpn_users(u)) + len(objects.list_dhcp_static_leases(u)) + len(objects.list_dns_records(u)) + objs = objects.list_objects(u, expand=True) + for o in objs: + if o['type'] == 'host_set': + assert 'ipaddr' in o + elif o['type'] == 'domain_set': + assert 'domain' in o + elif o['type'] == 'vpn_user': + assert 'ipaddr' in o + elif o['type'] == 'dhcp_static_lease': + assert 'ipaddr' in o + elif o['type'] == 'dns_record': + assert 'ipaddr' in o + objs = objects.list_objects(u, include_domain_sets=False) + for o in objs: + assert o['type'] != 'domain_set' + objs = objects.list_objects(u, singleton_only=True) + for o in objs: + if o['type'] == 'host_set': + assert o['singleton'] + assert o['subtype'] + +def test_get_reference_info(u): + ref = objects.get_info(u, "dhcp/ns_8dcab636") + assert ref['type'] == 'host' + assert ref['name'] == 'host2' + assert ref['id'] == 'ns_8dcab636' + assert ref['family'] == 'ipv4' + assert objects.get_info(u, "unknown") == None + ref2 = objects.get_info(u, "objects/ns_a7439990") + assert ref2['type'] == 'host' + assert ref2['name'] == 'h1v6' + assert ref2['id'] == 'ns_a7439990' + assert ref2['family'] == 'ipv6' + +def test_is_object_with_invalid_is(u): + assert objects.is_object_id(None) == False + assert objects.is_object_id(1) == False \ No newline at end of file diff --git a/packages/python3-nethsec/tests/test_ovpn.py b/packages/python3-nethsec/tests/test_ovpn.py new file mode 100644 index 000000000..a5424d4c2 --- /dev/null +++ b/packages/python3-nethsec/tests/test_ovpn.py @@ -0,0 +1,63 @@ +import pytest +import ipaddress +from nethsec import ovpn +from euci import EUci + +openvpn_db = """ +config openvpn 'custom_config' + option port '1194' + +config openvpn 'custom_config2' + option port '1300' + option server '10.10.10.0 255.255.255.0' + +config openvpn 'custom_config3' + option port '1301' + option server_bridge '10.8.0.4 255.255.255.0 10.8.0.128 10.8.0.254' + +config openvpn 'custom_config4' + option lport '1302' + +config openvpn 'custom_config5' + option lport '1303' + option ifconfig '10.7.0.1 10.7.0.2' +""" + +def _setup_db(tmp_path): + # setup fake db + with tmp_path.joinpath('openvpn').open('w') as fp: + fp.write(openvpn_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_to_cidr(): + assert ovpn.to_cidr("255.255.255.0") == 24 + +def test_to_netmask(): + assert ovpn.to_netmask(24) == "255.255.255.0" + +def test_is_used_network(tmp_path): + u = _setup_db(tmp_path) + assert ovpn.is_used_network(u, "10.10.10.0/24") == True + assert ovpn.is_used_network(u, "10.8.0.0/24") == True + assert ovpn.is_used_network(u, "10.7.0.0/24") == True + assert ovpn.is_used_network(u, "192.168.1.0/24") == False + +def test_generate_random_network(tmp_path): + u = _setup_db(tmp_path) + assert ipaddress.ip_network(ovpn.generate_random_network(u)).is_private + +def test_opt2cidr(): + assert ovpn.opt2cidr("192.168.1.0 255.255.255.0") == "192.168.1.0/24" + +def test_is_used_port(tmp_path): + u = _setup_db(tmp_path) + assert ovpn.is_used_port(u, 1194) == True + assert ovpn.is_used_port(u, 1300) == True + assert ovpn.is_used_port(u, 1301) == True + assert ovpn.is_used_port(u, 1302) == True + assert ovpn.is_used_port(u, 1303) == True + assert ovpn.is_used_port(u, 1200) == False + +def test_generate_random_port(tmp_path): + u = _setup_db(tmp_path) + assert type(ovpn.generate_random_port(u, 1200, 1210)) == type(1) \ No newline at end of file diff --git a/packages/python3-nethsec/tests/test_users.py b/packages/python3-nethsec/tests/test_users.py new file mode 100644 index 000000000..ab5b087e1 --- /dev/null +++ b/packages/python3-nethsec/tests/test_users.py @@ -0,0 +1,521 @@ +import pytest +from nethsec import utils, users +from euci import EUci, UciExceptionNotFound +from unittest.mock import MagicMock, patch + + +users_db = """ +config local 'main' + option description 'Main local database' + +config local 'second' + option description 'Secondary local database' + +config local 'third' + option description 'Third local database' + +config ldap 'ldap1' + option description 'Remote OpenLDAP server' + option uri 'ldaps://192.168.100.234' + option tls_reqcert 'never' + option base_dn 'dc=directory,dc=nh' + option user_dn 'ou=People,dc=directory,dc=nh' + option user_attr 'uid' + option user_display_attr 'cn' + option starttls '0' + option schema 'rfc2307' + +config ldap 'ldap2' + option description 'Remote OpenLDAP server' + option uri 'ldap://192.168.1.1' + option tls_reqcert 'always' + option base_dn 'dc=directory,dc=nh' + option user_dn 'ou=People,dc=directory,dc=nh' + option user_attr 'uid' + option user_display_attr 'cn' + option starttls '1' + option schema 'rfc2307' + +config ldap 'ad1' + option description 'Remote AD server' + option uri 'ldaps://ad.nethserver.org' + option tls_reqcert 'always' + option base_dn 'dc=ad,dc=nethserver,dc=org' + option user_dn 'cn=users,dc=ad,dc=nethserver,dc=org' + option user_display_attr 'cn' + option user_attr 'uid' + option starttls '0' + option schema 'ad' + +config user 'u1' + option name "goofy" + option database "main" + option description 'Goofy Doe' + list macaddr '52:54:00:9d:3d:e5' + list ipaddr '192.168.100.23' + list domain 'ns_goofy_name' + list host 'ns_goofy_pc' + option openvpn_ipaddr '10.9.9.38' + +config user 'u2' + option name "daisy" + option database "main" + option description "Daisy" + list ipaddr '192.168.100.22' + list ipaddr '2001:db8:3333:4444:5555:6666:7777:8888' + +config user 'u3' + option name "goofy" + option database "second" + option description 'Another Goofy' + +config user 'u4' + option name "goofy" + option database "ldap1" + +config user 'u5' + option name "john" + option database "ad1" + +config group 'g1' + option name "vip" + option database "main" + option description 'Very Important People' + list user 'goofy' + list user 'daisy' + +config group 'g2' + option name "vip" + option database "ldap1" +""" + +dhcp_db = """ +config host 'ns_goofy_pc' + option name 'goofypc' + option mac '00:00:8c:16:b3:bd' + option ip '192.168.100.36' + option dns '1' + +config domain 'ns_goofy_pc_nethserver_org' + option ip '192.168.100.30' + option name 'goofy.nethserver.org' + option description 'Goofy Workstation' +""" + +rpcd_db = "" + +def _setup_db(tmp_path): + # setup fake db + with tmp_path.joinpath('users').open('w') as fp: + fp.write(users_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('rpcd').open('w') as fp: + fp.write(rpcd_db) + + return EUci(confdir=tmp_path.as_posix()) + +def test_get_user_addresses(tmp_path): + u = _setup_db(tmp_path) + (ipv4, ipv6) = users.get_user_addresses(u, 'goofy') + for ip in ipv4: + assert(ip in ["192.168.100.36", "10.9.9.38", "192.168.100.30", "192.168.100.23"]) + (ipv4, ipv6) = users.get_user_addresses(u, 'daisy') + assert(ipv6 == ["2001:db8:3333:4444:5555:6666:7777:8888"]) + +def test_get_user_macs(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_user_macs(u, 'goofy') == ["52:54:00:9d:3d:e5"]) + assert(users.get_user_macs(u, 'daisy') == []) + +def test_get_group_addresses(tmp_path): + u = _setup_db(tmp_path) + (ipv4, ipv6) = users.get_group_addresses(u, 'vip') + for ip in ipv4: + assert(ip in ["192.168.100.36", "10.9.9.38", "192.168.100.30", "192.168.100.23", "192.168.100.22"]) + assert(ipv6 == ["2001:db8:3333:4444:5555:6666:7777:8888"]) + +def test_get_group_macs(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_group_macs(u, 'vip') == ["52:54:00:9d:3d:e5"]) + +def test_get_user_addresses(tmp_path): + u = _setup_db(tmp_path) + (ipv4, ipv6) = users.get_user_addresses(u, 'goofy') + for ip in ipv4: + assert(ip in ["192.168.100.36", "10.9.9.38", "192.168.100.30", "192.168.100.23"]) + (ipv4, ipv6) = users.get_user_addresses(u, 'daisy') + assert(ipv6 == ["2001:db8:3333:4444:5555:6666:7777:8888"]) + +def test_get_user_macs(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_user_macs(u, 'goofy') == ["52:54:00:9d:3d:e5"]) + assert(users.get_user_macs(u, 'daisy') == []) + +def test_get_group_addresses(tmp_path): + u = _setup_db(tmp_path) + (ipv4, ipv6) = users.get_group_addresses(u, 'vip') + for ip in ipv4: + assert(ip in ["192.168.100.36", "10.9.9.38", "192.168.100.30", "192.168.100.23", "192.168.100.22"]) + assert(ipv6 == ["2001:db8:3333:4444:5555:6666:7777:8888"]) + +def test_get_group_macs(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_group_macs(u, 'vip') == ["52:54:00:9d:3d:e5"]) + +def test_get_user_by_name(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_user_by_name(u, 'goofy') == { + "name": "goofy", + "database": "main", + "description": "Goofy Doe", + "macaddr": ["52:54:00:9d:3d:e5"], + "ipaddr": ["192.168.100.23"], + "domain": ["ns_goofy_name"], + "host": ["ns_goofy_pc"], + "openvpn_ipaddr": "10.9.9.38", + "local": True, + "admin": False, + "id": "u1", + }) + + assert(users.get_user_by_name(u, 'goofy', database="second") == { + "name": "goofy", + "description": "Another Goofy", + "database": "second", + "local": True, + "admin": False, + "id": "u3" + }) + + assert(users.get_user_by_name(u, 'daisy', database="second") == None) + +def test_get_group_by_name(tmp_path): + u = _setup_db(tmp_path) + assert(users.get_group_by_name(u, 'vip') == { + "name": "vip", + "database": "main", + "description": "Very Important People", + "user": ["goofy", "daisy"], + "local": True, + "id": "g1" + }) + +def test_list_databases(tmp_path): + db_list = users.list_databases(_setup_db(tmp_path)) + assert {"name": "main", "description": "Main local database", "type": "local"} in db_list + assert {"name": "second", "description": "Secondary local database", "type": "local"} in db_list + assert {"name": "ldap1", "description": "Remote OpenLDAP server", "type": "ldap", "schema": "rfc2307", "uri": "ldaps://192.168.100.234", "used": []} in db_list + assert {"name": "ad1", "description": "Remote AD server", "type": "ldap", "schema": "ad", "uri": "ldaps://ad.nethserver.org", "used": []} in db_list + +def test_add_local_database(tmp_path): + u = _setup_db(tmp_path) + users.add_local_database(u, "test", "Test database") + assert u.get('users', 'test') != None + assert u.get('users', 'test', 'description') == "Test database" + +def test_add_ldap_database(tmp_path): + u = _setup_db(tmp_path) + users.add_ldap_database(u, "testldap", "ldap://1.2.3.4", "ad", "dc=test,dc=org", "cn=users,dc=test,dc=org", "cn", "cn") + assert u.get('users', 'testldap') != None + assert u.get('users', 'testldap', 'uri') == "ldap://1.2.3.4" + assert u.get('users', 'testldap', 'schema') == "ad" + assert u.get('users', 'testldap', 'base_dn') == "dc=test,dc=org" + assert u.get('users', 'testldap', 'user_dn') == "cn=users,dc=test,dc=org" + assert u.get('users', 'testldap', 'user_attr') == "cn" + assert u.get('users', 'testldap', 'user_display_attr') == "cn" + assert u.get('users', 'testldap', 'start_tls') == "0" + assert u.get('users', 'testldap', 'tls_reqcert') == "never" + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap', 'user_bind_dn') + + users.add_ldap_database(u, "testldap2", "ldaps://server.nethserver.org", "rfc2307", "dc=test,dc=org", "ou=People,dc=test,dc=org", "uid", "displayName", start_tls=True, tls_reqcert="always", description="mydesc") + assert u.get('users', 'testldap2') != None + assert u.get('users', 'testldap2', 'description') == "mydesc" + assert u.get('users', 'testldap2', 'uri') == "ldaps://server.nethserver.org" + assert u.get('users', 'testldap2', 'schema') == "rfc2307" + assert u.get('users', 'testldap2', 'base_dn') == "dc=test,dc=org" + assert u.get('users', 'testldap2', 'user_dn') == "ou=People,dc=test,dc=org" + assert u.get('users', 'testldap2', 'user_attr') == "uid" + assert u.get('users', 'testldap2', 'user_display_attr') == "displayName" + assert u.get('users', 'testldap2', 'start_tls') == "1" + assert u.get('users', 'testldap2', 'tls_reqcert') == "always" + + users.add_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test,dc=org", "ou=People,dc=test,dc=org", "uid", "displayName", start_tls=True, tls_reqcert="always", description="mydesc", bind_dn="cn=admin,dc=test,dc=org", bind_password="12=34", user_bind_dn="%u@test.org") + assert u.get('users', 'testldap3') != None + assert u.get('users', 'testldap3', 'bind_dn') == "cn=admin,dc=test,dc=org" + assert u.get('users', 'testldap3', 'bind_password') == "12=34" + assert u.get('users', 'testldap3', 'user_bind_dn') == "%u@test.org" + + +def test_list_users(tmp_path): + u = _setup_db(tmp_path) + user_list = users.list_users(_setup_db(tmp_path), "second") + assert(users.get_user_by_name(u, 'goofy', "second") in user_list) + +def test_edit_ldap_database(tmp_path): + u = _setup_db(tmp_path) + users.edit_ldap_database(u, "testldap2", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2") + assert u.get_all('users', 'testldap2') == { + "description": "mydesc2", + "uri": "ldaps://server.nethserver.org", + "schema": "rfc2307", + "base_dn": "dc=test2,dc=org2", + "user_dn": "dc=test2,dc=org2", + "user_attr": "uid", + "user_display_attr": "cn", + "start_tls": "0", + "tls_reqcert": "never" + } + + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap3', 'bind_dn') + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap3', 'bind_password') + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2", bind_dn="cn=admin2,dc=test,dc=org", bind_password="4567", user_bind_dn="%u@test2.org2") + assert u.get('users', 'testldap3', 'bind_dn') == "cn=admin2,dc=test,dc=org" + assert u.get('users', 'testldap3', 'bind_password') == "4567" + assert u.get('users', 'testldap3', 'user_bind_dn') == "%u@test2.org2" + # check if user_bind_dn is deleted if not passed as parameter + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2", bind_dn="cn=admin2,dc=test,dc=org", bind_password="4567") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap3', 'user_bind_dn') + # add back user_bind_dn, check if it's removed if passed as empty string + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2", bind_dn="cn=admin2,dc=test,dc=org", bind_password="4567", user_bind_dn="%u@test3.org3") + assert u.get('users', 'testldap3', 'user_bind_dn') == "%u@test3.org3" + users.edit_ldap_database(u, "testldap3", "ldaps://server.nethserver.org", "rfc2307", "dc=test2,dc=org2", "dc=test2,dc=org2", "uid", "cn", start_tls=False, tls_reqcert="never", description="mydesc2", bind_dn="cn=admin2,dc=test,dc=org", bind_password="4567", user_bind_dn="") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'testldap3', 'user_bind_dn') + + + +def test_delete_ldap_database(tmp_path): + u = _setup_db(tmp_path) + users.delete_ldap_database(u, "ldap1") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'ldap1') + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'u4') + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'g2') + +def test_edit_local_database(tmp_path): + u = _setup_db(tmp_path) + users.edit_local_database(u, "main", "Main local database2") + assert u.get('users', 'main', 'description') == "Main local database2" + +def test_delete_local_database(tmp_path): + u = _setup_db(tmp_path) + users.delete_local_database(u, "main") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', 'main') == None + with pytest.raises(UciExceptionNotFound) as e: + u.get('users', 'u1') + with pytest.raises(UciExceptionNotFound) as e: + u.get('users', 'u2') + with pytest.raises(UciExceptionNotFound) as e: + u.get('users', 'g1') + +def test_add_local_user(tmp_path): + u = _setup_db(tmp_path) + id = users.add_local_user(u, "t1", password="nethesis", description="mydesc", database="second", extra_fields={"openvpn_ipaddr": "1.2.3.4"}) + user = users.get_user_by_name(u, "t1", "second") + assert user != None + assert users.check_password("nethesis", user.pop("password")) + assert user == { + "name": "t1", + "description": "mydesc", + "database": "second", + "local": True, + "admin": False, + "id": id, + "openvpn_ipaddr": "1.2.3.4" + } + +def test_edit_local_user(tmp_path): + u = _setup_db(tmp_path) + id = users.edit_local_user(u, "t1", password="pass2", description='mydesc2', database="second", extra_fields={"openvpn_ipaddr": "1.2.3.5", "openvpn_enabled": "1"}) + user = users.get_user_by_name(u, "t1", "second") + assert users.check_password("pass2", user.pop("password")) + assert user == { + "name": "t1", + "description": "mydesc2", + "database": "second", + "local": True, + "admin": False, + "id": id, + "openvpn_ipaddr": "1.2.3.5", + "openvpn_enabled": "1" + } + # make sure description is not changed if not passed as parameter + id = users.edit_local_user(u, "t1", password="pass2", database="second", extra_fields={"openvpn_ipaddr": "1.2.3.5", "openvpn_enabled": "1"}) + assert u.get('users', id, 'description') == 'mydesc2' + id = users.edit_local_user(u, "t1", password="pass2", database="second", extra_fields={"openvpn_enabled": "1"}) + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', id, 'openvpn_ipaddr') + +def test_add_local_group(tmp_path): + u = _setup_db(tmp_path) + id = users.add_local_group(u, "group3", ["goofy", "daisy"], "mydesc", database="second") + assert u.get_all('users', id) == { + "name": "group3", + "description": "mydesc", + "database": "second", + "user": ("goofy", "daisy"), + } + +def test_edit_local_group(tmp_path): + u = _setup_db(tmp_path) + id = users.edit_local_group(u, "group3", ["goofy"], "mydesc2", database="second") + assert u.get('users', id, 'description') == 'mydesc2' + assert u.get('users', id, 'user', list=True) == ('goofy',) + +def test_delete_local_user(tmp_path): + u = _setup_db(tmp_path) + assert users.delete_local_user(u, "t1", database="second") + assert users.get_user_by_name(u, "t1", "second") == None + assert users.delete_local_user(u, "goofy", database="second") + group = users.get_group_by_name(u, "group3", "second") + assert group.get('user',[]) == [] + +def test_delete_local_group(tmp_path): + u = _setup_db(tmp_path) + group = users.get_group_by_name(u, "group3", "second") + users.delete_local_group(u, "group3", database="second") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', group['id']) + +def test_add_remote_user(tmp_path): + u = _setup_db(tmp_path) + id = users.add_remote_user(u, "john", "ldap2", extra_fields={"openvpn_ipaddr": "1.2.3.4"}) + u.get('users', id, 'name') == "john" + u.get('users', id, 'database') == "ldap2" + u.get('users', id, 'openvpn_ipaddr') == "1.2.3.4" + +def test_edit_remote_user(tmp_path): + u = _setup_db(tmp_path) + id = users.edit_remote_user(u, "john", "ldap2", extra_fields={"openvpn_enabled": "1"}) + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', id, 'openvpn_ipaddr') + u.get('users', id, 'openvpn_enabled') == "1" + +def test_delete_remote_user(tmp_path): + u = _setup_db(tmp_path) + user = users.get_user_by_name(u, "john", "ldap2") + users.delete_remote_user(u, "john", database="ldap2") + with pytest.raises(UciExceptionNotFound) as e: + assert u.get('users', user['id']) + +def test_shadow_password(): + shadow = users.shadow_password("test") + assert(users.check_password("test", shadow)) + +def test_ldif2users(): + ldif_data = """ +dn: uid=admin,ou=People,dc=directory,dc=nh +uid: admin +cn: admin + +dn: uid=pluto,ou=People,dc=directory,dc=nh +uid: pluto +cn: Pluto Rossi + +# pagedresults: cookie= +""" + assert users.ldif2users(ldif_data) == [{"name": "admin", "description": "admin"},{"name":"pluto", "description": "Pluto Rossi"}] + + ldif_data = """ +# John Doe, Users, domtest1.local +dn: CN=John Doe,CN=Users,DC=domtest1,DC=local +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: user +cn: John Doe +sn: doe +description: myoffice +physicalDeliveryOfficeName: myoffice +givenName: john +distinguishedName: CN=John Doe,CN=Users,DC=domtest1,DC=local +instanceType: 4 +whenCreated: 20240605091444.0Z +whenChanged: 20240702131054.0Z +displayName: John Doe +uSNCreated: 45212 +uSNChanged: 49102 +name: John Doe +objectGUID:: 0JQMYe1R+0uziXQFbmQoJg== +userAccountControl: 512 +badPwdCount: 0 +codePage: 0 +countryCode: 0 +badPasswordTime: 133621586649855275 +lastLogoff: 0 +lastLogon: 133621586960949063 +pwdLastSet: 133632746540757598 +primaryGroupID: 513 +objectSid:: AQUAAAAAAAUVAAAAvh3X9UPJVVW6ljm9VwQAAA== +accountExpires: 9223372036854775807 +logonCount: 0 +sAMAccountName: john.doe +sAMAccountType: 805306368 +userPrincipalName: john.doe@domtest1.local +objectCategory: CN=Person,CN=Schema,CN=Configuration,DC=domtest1,DC=local +dSCorePropagationData: 16010101000000.0Z +""" + assert users.ldif2users(ldif_data, 'sAMAccountName', 'displayName') == [{"name": "john.doe", "description": "John Doe"}] + +def test_set_admin_user(tmp_path): + u = _setup_db(tmp_path) + users.add_local_user(u, "admin2", password="nethesis", description="mydesc", database="third") + admin_id_rpcd = users.set_admin(u, "admin2", "third") + for user in utils.get_all_by_type(u, "rpcd", "login"): + if user == admin_id_rpcd: + assert u.get('rpcd', admin_id_rpcd, "username") == "admin2" + assert users.check_password("nethesis", u.get('rpcd', admin_id_rpcd, "password")) + +def test_is_admin(tmp_path): + u = _setup_db(tmp_path) + assert users.is_admin(u, "admin2") + user = users.get_user_by_name(u, "admin2", 'third') + assert user.get("admin") + +def test_change_admin_password(tmp_path): + u = _setup_db(tmp_path) + local_id = users.edit_local_user(u, "admin2", password="nethesis", description="mydesc", database="third") + logins = utils.get_all_by_type(u, "rpcd", "login") + for l in logins: + if logins[l].get("username") == "admin2": + assert logins[l].get("password") == u.get("users", local_id, "password") + +def test_remove_admin_user(tmp_path): + u = _setup_db(tmp_path) + users.remove_admin(u, "admin2") + found = False + for user in utils.get_all_by_type(u, "rpcd", "login"): + if user.get("username") == "admin2": + found = True + assert not found + +def test_get_database(tmp_path): + u = _setup_db(tmp_path) + assert users.get_database(u, "third") == { + "name": "third", + "description": "Third local database", + "type": "local" + } + + assert users.get_database(u, "ad1") == { + "name": "ad1", + "description": "Remote AD server", + "type": "ldap", + "schema": "ad", + "uri": "ldaps://ad.nethserver.org", + "tls_reqcert": "always", + "base_dn": "dc=ad,dc=nethserver,dc=org", + "user_dn": "cn=users,dc=ad,dc=nethserver,dc=org", + "user_attr": "uid", + "user_display_attr": "cn", + "starttls": "0" + } diff --git a/packages/python3-nethsec/tests/test_utils.py b/packages/python3-nethsec/tests/test_utils.py new file mode 100644 index 000000000..0c9d4abc5 --- /dev/null +++ b/packages/python3-nethsec/tests/test_utils.py @@ -0,0 +1,229 @@ +import pytest +from nethsec import utils +from euci import EUci +from unittest.mock import MagicMock, patch + +# Setup fake ip command output +ip_json='[{"ifindex":9,"ifname":"vnet3","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"virbr2","operstate":"UNKNOWN","group":"default","txqlen":1000,"link_type":"ether","address":"fe:62:31:19:0b:29","broadcast":"ff:ff:ff:ff:ff:ff","addr_info":[{"family":"inet6","local":"fe80::fc62:31ff:fe19:b29","prefixlen":64,"scope":"link","valid_life_time":4294967295,"preferred_life_time":4294967295}]},{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":3,"ifname":"eth1","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":4,"ifname":"eth2","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:75:1c:c1","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":5,"ifname":"eth3","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":6,"ifname":"ifb-dns","flags":["BROADCAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":32,"link_type":"ether","address":"72:79:65:12:07:07","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":7,"ifname":"br-lan","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":9,"ifname":"bond-bond1","flags":["BROADCAST","MULTICAST","MASTER","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":8,"ifname":"tuntunsubnet","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":500,"link_type":"none"}, {"ifindex":69,"ifname":"pppoe-w1","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1492,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":3,"link_type":"ppp"},{"ifindex":20,"link":"eth1","ifname":"eth1.4","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"}]' +mock_ip_stdout = MagicMock() +mock_ip_stdout.configure_mock(**{"stdout": ip_json}) + +test_db = """ +config mytype section1 + option name 'myname1' + option opt2 'value2' + +config mytype section2 + option name 'myname2' + list opt1 'val1' + list opt1 'val2' + option opt2 'value2' + +config mytype2 section3 + option name 'myname3' +""" + +firewall_db = """ +config zone wan1 + option name 'wan' + list network 'wan' + list network 'wan6' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + list device 'eth2.1' + +config zone + option name 'lan' + list network 'lan' + list network 'br-lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + list device 'tunrw1' + list device 'ipsec2' +""" + +network_db = """ +config interface lan + option device 'vnet3' + option proto 'static' + +config device + option name 'br-lan' + option type 'bridge' + list ports 'eth0' + +config device + option type '8021q' + option ifname 'eth2' + option vid '1' + option name 'eth2.1' + +config interface 'wan' + option device 'eth1' + option proto 'dhcp' + +config interface 'wan6' + option device 'eth1' + option proto 'dhcpv6' + +config interface 'bond1' + option proto 'bonding' + option ipaddr '10.0.0.22' + option netmask '255.255.255.0' + list slaves 'eth3' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + option all_slaves_active '0' + option link_monitoring 'off' + +config interface 'bond2' + option proto 'bonding' + option ipaddr '10.11.11.22' + option netmask '255.255.255.0' + option bonding_policy 'balance-rr' + option packets_per_slave '1' + list slaves 'eth3.2' + +config device + option ifname 'eth0' + option name 'eth0.1' + option type '8021q' + option vid '45' + +config device 'ns_29a705ea' + option name 'eth3.2' + option type '8021q' + option ifname 'eth3' + option vid '2' +""" + +dedalo_db = """ +config dedalo 'config' + option disabled '0' + option interface 'eth2' +""" + +def _setup_db(tmp_path): + # setup fake db + with tmp_path.joinpath('test').open('w') as fp: + fp.write(test_db) + with tmp_path.joinpath('network').open('w') as fp: + fp.write(network_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('dedalo').open('w') as fp: + fp.write(dedalo_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_sanitize(): + assert utils.sanitize("good") == "good" + assert utils.sanitize("with-dash") == "with_dash" + assert utils.sanitize('$%_()') == '____' + assert utils.sanitize('UPPER') == 'UPPER' + assert utils.sanitize('numb3r') == 'numb3r' + assert utils.sanitize('newline\n') == 'newline' + assert utils.sanitize('newline\r') == 'newline' + +def test_get_id(): + assert utils.get_id('no-good') == 'ns_no_good' + assert utils.get_id('nospace ') == 'ns_nospace' + assert utils.get_id('t1234') == 'ns_t1234' + # str with 97 chars + long_str = "ihTSEf2Y5rl8TX96pWFFPMty9LFgH3GezhVueGoDB6_aaIFhDSKe1ZR64cV41iSVhfrm5wJCUPFfMGx2fBZyhDIW9cl9SCI43" + assert utils.get_id(long_str) == f'ns_{long_str}' + assert utils.get_id(long_str+"123") == f'ns_{long_str}' + +def test_get_id_length(): + assert utils.get_id("123456789012345", 15) == "ns_123456789012" + +def test_get_all_by_type(tmp_path): + u = _setup_db(tmp_path) + records = utils.get_all_by_type(u, 'test', 'mytype') + assert records != None + assert 'section1' in records.keys() + assert 'section2' in records.keys() + assert 'section3' not in records.keys() + assert records['section1']['name'] == 'myname1' + assert records['section2']['name'] == 'myname2' + assert records['section2']['opt1'] == ('val1', 'val2') + +@patch("nethsec.utils.subprocess.run") +def test_get_device_name_valid(mock_run): + # setup mock + mock_run.return_value = mock_ip_stdout + + assert utils.get_device_name("fe:62:31:19:0b:29") == 'vnet3' + +def test_get_device_name_not_valid(): + assert utils.get_device_name("aa:bb:cc:dd:66:55") == None + +@patch("nethsec.utils.subprocess.run") +def test_get_interface_from_mac(mock_run2, tmp_path): + # setup mock + mock_run2.return_value = mock_ip_stdout + u = _setup_db(tmp_path) + + assert utils.get_interface_from_mac(u, "fe:62:31:19:0b:29") == 'lan' + assert utils.get_interface_from_mac(u, "aa:bb:cc:dd:66:55") == None + +def test_get_interface_from_device(tmp_path): + u = _setup_db(tmp_path) + + assert utils.get_interface_from_device(u, "vnet3") == 'lan' + assert utils.get_interface_from_device(u, "vnet4") == None + +def test_get_all_by_option(tmp_path): + u = _setup_db(tmp_path) + return_map = {"section1": u.get_all("test", "section1"), "section2": u.get_all("test", "section2")} + assert(utils.get_all_by_option(u, 'test', 'opt2', 'value2') == return_map) + assert(utils.get_all_by_option(u, 'test', 'opt2', 'value2', deep = False) == list(return_map.keys())) + +def test_get_all_wan_devices(tmp_path): + u = _setup_db(tmp_path) + assert(set(utils.get_all_wan_devices(u)) == set(['eth1', 'eth2.1'])) + +def test_get_all_lan_devices(tmp_path): + u = _setup_db(tmp_path) + assert(set(utils.get_all_lan_devices(u)) == set(['vnet3'])) + +def test_get_random_id(): + id1 = utils.get_random_id() + id2 = utils.get_random_id() + assert len(id1) == 11 + assert id1[0:3] == "ns_" + assert id1 != id2 + +def test_error(): + obj = utils.generic_error("my Error ") + assert obj == {"error": "my_error"} + +def test_validation_error(): + er1 = utils.validation_error("param1", "my Error ", "val1") + assert er1 == {"validation": {"errors": [{"parameter": "param1", "message": "my_error", "value": "val1"}]}} + er2 = utils.validation_error("param1") + assert er2 == {"validation": {"errors": [{"parameter": "param1", "message": "", "value": ""}]}} + +def test_validation_errors(): + er1 = utils.validation_errors([ + ["param1", " my Error ", "val1"], + ["param2", " invalid_value ", False], + ]) + assert er1 == {"validation": {"errors": [{"parameter": "param1", "message": "my_error", "value": "val1"}, {"parameter": "param2", "message": "invalid_value", "value": False}]}} + +@patch("nethsec.utils.subprocess.run") +def test_get_unassigned_devices(mock_run_ip, tmp_path): + # setup mock + mock_run_ip.return_value = mock_ip_stdout + + u = _setup_db(tmp_path) + assert utils.get_unassigned_devices(u) == ['eth0.1'] + +def test_check_password(): + password = "my_password" + shadow = utils.shadow_password(password) + assert utils.check_password(password, shadow) == True + assert utils.check_password("wrong_password", shadow) == False