diff --git a/.ci/unit_tests/Python_Engine_Test/Compute/Install.cs b/.ci/unit_tests/Python_Engine_Test/Compute/Install.cs new file mode 100644 index 00000000..c44df2d6 --- /dev/null +++ b/.ci/unit_tests/Python_Engine_Test/Compute/Install.cs @@ -0,0 +1,228 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using FluentAssertions; +using EngineCompute = BH.Engine.Python.Compute; +using EngineQuery = BH.Engine.Python.Query; + +namespace BH.Tests.Engine.Python.Compute +{ + [NonParallelizable] + public class InstallTests : InstallRemoveTestBase + { + [Test] + public void InstallPythonToolkitVenv_InstallsAndReloadsToolkitEnvironment() + { + DirectoryInfo? repository = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (repository != null && !File.Exists(Path.Combine(repository.FullName, "Python_Engine", "Python", "pyproject.toml"))) + repository = repository.Parent; + repository.Should().NotBeNull("the toolkit sources must be available"); + string source = Path.Combine(repository!.FullName, "Python_Engine", "Python"); + string directory = Path.Combine(EngineQuery.DirectoryEnvironments(), EngineQuery.ToolkitName()); + + WithIsolatedDirectory(directory, () => + { + foreach (string file in new[] { "pyproject.toml", "README.md" }) + File.Copy(Path.Combine(source, file), Path.Combine(directory, file)); + string sourceFolder = Path.Combine(source, "src"); + foreach (string file in Directory.EnumerateFiles(sourceFolder, "*", SearchOption.AllDirectories)) + { + string relative = Path.GetRelativePath(sourceFolder, file); + if (relative.Split(Path.DirectorySeparatorChar).Contains("__pycache__")) + continue; + string destination = Path.Combine(directory, "src", relative); + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + File.Copy(file, destination); + } + + try + { + var venv = EngineCompute.InstallPythonToolkitVenv(run: true, reload: false); + + venv.Should().NotBeNull(); + venv.Name.Should().Be(EngineQuery.ToolkitName()); + venv.Executable.Should().Be(Path.Combine(directory, ".venv", "Scripts", "python.exe")); + File.Exists(venv.Executable).Should().BeTrue(); + EngineCompute.RunPythonString(venv, + "import importlib.metadata; print(importlib.metadata.version('python_toolkit'))") + .Should().Be(BH.Engine.Base.Query.BHoMVersion()); + File.Exists(Path.Combine(m_Root, "jupyter", "kernels", venv.Name.ToLowerInvariant(), "kernel.json")).Should().BeTrue(); + EngineCompute.InstallPythonToolkitVenv(run: true, reload: true).Executable.Should().Be(venv.Executable); + AssertNoErrors(); + } + finally + { + string kernel = Path.Combine(m_Root, "jupyter", "kernels", EngineQuery.ToolkitName().ToLowerInvariant(), "kernel.json"); + if (File.Exists(kernel) && EngineQuery.VirtualEnvironmentExists(EngineQuery.ToolkitName())) + EngineCompute.RemoveJupyterKernel(EngineQuery.ToolkitName(), run: true); + } + }); + } + + [Test] + public void InstallUV_InstallsMissingTools() + { + PrepareToolPackages(); + + EngineCompute.InstallUV(run: true); + + string tools = Run(BH.Engine.Python.Query.UVExecutable(), "tool", "list"); + foreach (string tool in new[] { "ruff", "jupyterlab", "sphinx" }) + tools.Should().Contain(tool + " v1.0.0"); + AssertNoErrors(); + } + + [Test] + public void InstallUV_SkipsExistingTools() + { + PrepareToolPackages(); + EngineCompute.InstallUV(run: true); + string[] receipts = Directory.GetFiles(Path.Combine(m_Root, "tools"), "uv-receipt.toml", SearchOption.AllDirectories); + receipts.Should().HaveCount(3); + var timestamps = receipts.ToDictionary(path => path, File.GetLastWriteTimeUtc); + + EngineCompute.InstallUV(run: true); + + Environment.GetEnvironmentVariable("PATH")!.Split(Path.PathSeparator)[0] + .Should().Be(BH.Engine.Python.Query.DirectoryTools()); + BH.Engine.Python.Query.UVExecutable().Should() + .Be(Path.Combine(BH.Engine.Python.Query.DirectoryTools(), "uv.exe")); + foreach (var receipt in timestamps) + File.GetLastWriteTimeUtc(receipt.Key).Should().Be(receipt.Value); + AssertNoErrors(); + } + + [Test] + public void InstallPackages_SkipsSatisfiedVersion() + { + PreparePackageVersions(); + EngineCompute.InstallPackages(m_Environment, new() { "bhom-repeat-package==1.0.0" }, run: true); + string metadata = PackageMetadata("bhom_repeat_package", "1.0.0"); + DateTime timestamp = File.GetLastWriteTimeUtc(metadata); + + EngineCompute.InstallPackages(m_Environment, new() { "bhom-repeat-package>=1,<2" }, run: true); + + File.GetLastWriteTimeUtc(metadata).Should().Be(timestamp); + AssertNoErrors(); + } + + [Test] + public void InstallPackages_ChangesUnsatisfiedVersion() + { + PreparePackageVersions(); + EngineCompute.InstallPackages(m_Environment, new() { "bhom-repeat-package==1.0.0" }, run: true); + + EngineCompute.InstallPackages(m_Environment, new() { "bhom-repeat-package==2.0.0" }, run: true); + + File.Exists(PackageMetadata("bhom_repeat_package", "2.0.0")).Should().BeTrue(); + File.Exists(PackageMetadata("bhom_repeat_package", "1.0.0")).Should().BeFalse(); + AssertNoErrors(); + } + + [Test] + public void InstallPackages_RegistersAlreadyInstalledProjectDependency() + { + PreparePackageVersions(); + EngineCompute.InstallPackages(m_Environment, new() { "bhom-repeat-package==1.0.0" }, run: true); + string metadata = PackageMetadata("bhom_repeat_package", "1.0.0"); + DateTime timestamp = File.GetLastWriteTimeUtc(metadata); + CreateProject(); + + EngineCompute.InstallPackages(m_Environment, new() { "bhom-repeat-package==1.0.0" }, run: true); + + File.ReadAllText(Path.Combine(m_Root, "pyproject.toml")).Should().Contain("bhom-repeat-package==1.0.0"); + File.GetLastWriteTimeUtc(metadata).Should().Be(timestamp); + AssertNoErrors(); + } + + [Test] + public void InstallPackageLocal_SkipsSameUnchangedEditableSource() + { + string source = CreateLocalPackage("source with spaces"); + EngineCompute.InstallPackageLocal(m_Environment, source, run: true); + string metadata = PackageMetadata("bhom_local_package", "1.0.0"); + File.Exists(metadata).Should().BeTrue(); + DateTime timestamp = File.GetLastWriteTimeUtc(metadata); + + EngineCompute.InstallPackageLocal(m_Environment, source, run: true); + + File.GetLastWriteTimeUtc(metadata).Should().Be(timestamp); + AssertNoErrors(); + } + + [Test] + public void InstallPackageLocal_ReplacesDifferentEditableSource() + { + string first = CreateLocalPackage("first"); + string second = CreateLocalPackage("second"); + EngineCompute.InstallPackageLocal(m_Environment, first, run: true); + + EngineCompute.InstallPackageLocal(m_Environment, second, run: true); + + string directUrl = Path.Combine(Path.GetDirectoryName(PackageMetadata("bhom_local_package", "1.0.0"))!, "direct_url.json"); + File.ReadAllText(directUrl).Should().Contain(new Uri(second + Path.DirectorySeparatorChar).AbsoluteUri.TrimEnd('/')); + AssertNoErrors(); + } + + [Test] + public void InstallPackageLocal_UpdatesChangedVersion() + { + string source = CreateLocalPackage("local"); + EngineCompute.InstallPackageLocal(m_Environment, source, run: true); + string project = Path.Combine(source, "pyproject.toml"); + File.WriteAllText(project, File.ReadAllText(project).Replace("1.0.0", "2.0.0")); + + EngineCompute.InstallPackageLocal(m_Environment, source, run: true); + + File.Exists(PackageMetadata("bhom_local_package", "2.0.0")).Should().BeTrue(); + File.Exists(PackageMetadata("bhom_local_package", "1.0.0")).Should().BeFalse(); + AssertNoErrors(); + } + + [Test] + public void InstallPackageLocal_RegistersProjectDependency() + { + string source = CreateLocalPackage("local"); + EngineCompute.InstallPackageLocal(m_Environment, source, run: true); + CreateProject(); + + EngineCompute.InstallPackageLocal(m_Environment, source, run: true); + + File.ReadAllText(Path.Combine(m_Root, "pyproject.toml")).Should().Contain("bhom-local-package"); + AssertNoErrors(); + } + + [Test] + public void InstallJupyterKernel_SkipsExistingRegistration() + { + EngineCompute.InstallJupyterKernel(m_Environment); + string kernel = Path.Combine(m_Root, "jupyter", "kernels", m_Environment.Name, "kernel.json"); + File.Exists(kernel).Should().BeTrue(); + DateTime timestamp = File.GetLastWriteTimeUtc(kernel); + + EngineCompute.InstallJupyterKernel(m_Environment); + + File.GetLastWriteTimeUtc(kernel).Should().Be(timestamp); + AssertNoErrors(); + } + } +} diff --git a/.ci/unit_tests/Python_Engine_Test/Compute/InstallRemoveTestBase.cs b/.ci/unit_tests/Python_Engine_Test/Compute/InstallRemoveTestBase.cs new file mode 100644 index 00000000..0c112576 --- /dev/null +++ b/.ci/unit_tests/Python_Engine_Test/Compute/InstallRemoveTestBase.cs @@ -0,0 +1,221 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using FluentAssertions; +using BH.oM.Python; +using BH.oM.Base.Debugging; +using System.Diagnostics; +using System.IO.Compression; +using EngineQuery = BH.Engine.Python.Query; + +namespace BH.Tests.Engine.Python.Compute +{ + public abstract class InstallRemoveTestBase + { + protected string m_Root = null!; + protected PythonEnvironment m_Environment = null!; + private readonly Dictionary m_OriginalVariables = new(); + + [OneTimeSetUp] + public void EnsureManagedUV() + { + string? originalPath = Environment.GetEnvironmentVariable("PATH"); + try + { + BH.Engine.Base.Compute.ClearCurrentEvents(); + BH.Engine.Python.Compute.InstallUV(run: true); + File.Exists(EngineQuery.UVExecutable()).Should().BeTrue(); + AssertNoErrors(); + } + finally + { + Environment.SetEnvironmentVariable("PATH", originalPath); + BH.Engine.Base.Compute.ClearCurrentEvents(); + } + } + + [SetUp] + public void SetUp() + { + string name = "bhom_install_remove_" + Guid.NewGuid().ToString("N"); + m_Root = Path.Combine(EngineQuery.DirectoryEnvironments(), name); + Directory.CreateDirectory(m_Root); + m_Environment = new PythonEnvironment + { + Name = name, + Executable = Path.Combine(m_Root, ".venv", "Scripts", "python.exe") + }; + Run(EngineQuery.UVExecutable(), "venv", "--no-project", "--no-python-downloads", Path.Combine(m_Root, ".venv")); + SetVariable("JUPYTER_DATA_DIR", Path.Combine(m_Root, "jupyter")); + SetVariable("JUPYTER_PATH", Path.Combine(m_Root, "jupyter")); + SetVariable("UV_TOOL_DIR", Path.Combine(m_Root, "tools")); + SetVariable("UV_TOOL_BIN_DIR", Path.Combine(m_Root, "bin")); + SetVariable("UV_CACHE_DIR", Path.Combine(m_Root, "cache")); + string conflictingUV = Path.Combine(m_Root, "conflicting-uv"); + Directory.CreateDirectory(conflictingUV); + File.WriteAllText(Path.Combine(conflictingUV, "uv.cmd"), "@echo UV from PATH must not be used 1>&2\r\n@exit /b 99\r\n"); + SetVariable("PATH", conflictingUV + Path.PathSeparator + Environment.GetEnvironmentVariable("PATH")); + BH.Engine.Base.Compute.ClearCurrentEvents(); + } + + [TearDown] + public void TearDown() + { + foreach (var variable in m_OriginalVariables) + Environment.SetEnvironmentVariable(variable.Key, variable.Value); + m_OriginalVariables.Clear(); + if (Directory.Exists(m_Root)) + Directory.Delete(m_Root, true); + BH.Engine.Base.Compute.ClearCurrentEvents(); + } + + protected void PrepareToolPackages() + { + foreach (string tool in new[] { "ruff", "jupyterlab", "sphinx" }) + CreateWheel(tool, "1.0.0"); + SetVariable("UV_FIND_LINKS", Path.Combine(m_Root, "wheels")); + SetVariable("UV_OFFLINE", "true"); + } + + protected void PreparePackageVersions() + { + CreateWheel("bhom_repeat_package", "1.0.0"); + CreateWheel("bhom_repeat_package", "2.0.0"); + SetVariable("UV_FIND_LINKS", Path.Combine(m_Root, "wheels")); + SetVariable("UV_OFFLINE", "true"); + } + + private void CreateWheel(string name, string version) + { + string wheels = Path.Combine(m_Root, "wheels"); + Directory.CreateDirectory(wheels); + using var archive = ZipFile.Open(Path.Combine(wheels, $"{name}-{version}-py3-none-any.whl"), ZipArchiveMode.Create); + string info = $"{name}-{version}.dist-info"; + var files = new Dictionary + { + [$"{name}.py"] = "def main():\n pass\n", + [$"{info}/METADATA"] = $"Metadata-Version: 2.1\nName: {name.Replace('_', '-')}\nVersion: {version}\n", + [$"{info}/WHEEL"] = "Wheel-Version: 1.0\nGenerator: BHoM-tests\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + [$"{info}/entry_points.txt"] = $"[console_scripts]\n{name} = {name}:main\n" + }; + files[$"{info}/RECORD"] = string.Join("\n", files.Keys.Select(path => path + ",,")) + $"\n{info}/RECORD,,\n"; + foreach (var file in files) + { + using var writer = new StreamWriter(archive.CreateEntry(file.Key).Open()); + writer.Write(file.Value); + } + } + + protected string CreateLocalPackage(string folder) + { + string source = Path.Combine(m_Root, folder); + Directory.CreateDirectory(source); + File.WriteAllText(Path.Combine(source, "pyproject.toml"), + "[build-system]\nrequires = [\"setuptools\"]\nbuild-backend = \"setuptools.build_meta\"\n" + + "[project]\nname = \"bhom-local-package\"\nversion = \"1.0.0\"\n"); + File.WriteAllText(Path.Combine(source, "bhom_local_package.py"), "VALUE = 1\n"); + return source; + } + + protected void CreateProject() + { + File.WriteAllText(Path.Combine(m_Root, "pyproject.toml"), + "[project]\nname = \"bhom-test-project\"\nversion = \"1.0.0\"\nrequires-python = \">=3.10\"\ndependencies = []\n"); + } + + protected string PackageMetadata(string name, string version) + { + return Path.Combine(m_Root, ".venv", "Lib", "site-packages", $"{name}-{version}.dist-info", "METADATA"); + } + + protected void SetVariable(string name, string value) + { + if (!m_OriginalVariables.ContainsKey(name)) + m_OriginalVariables.Add(name, Environment.GetEnvironmentVariable(name)); + Environment.SetEnvironmentVariable(name, value); + } + + protected static void WithIsolatedDirectory(string directory, Action test) + { + string backup = directory + ".test-backup-" + Guid.NewGuid().ToString("N"); + bool existed = Directory.Exists(directory); + if (existed) + Directory.Move(directory, backup); + try + { + Directory.CreateDirectory(directory); + test(); + } + finally + { + try + { + if (Directory.Exists(directory)) + Directory.Delete(directory, true); + } + finally + { + if (existed) + Directory.Move(backup, directory); + } + } + } + + protected static void AssertNoErrors() + { + BH.Engine.Base.Query.CurrentEvents().Where(e => e.Type == EventType.Error).Select(e => e.Message).Should().BeEmpty(); + } + + protected static void AssertWarningWithoutErrors() + { + AssertNoErrors(); + BH.Engine.Base.Query.CurrentEvents().Should().Contain(e => e.Type == EventType.Warning); + } + + protected static string Run(string executable, params string[] arguments) + { + using var process = new Process + { + StartInfo = new ProcessStartInfo(executable) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + } + }; + foreach (string argument in arguments) + process.StartInfo.ArgumentList.Add(argument); + process.Start(); + Task stdout = process.StandardOutput.ReadToEndAsync(); + Task stderr = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(120000)) + { + process.Kill(entireProcessTree: true); + Assert.Fail($"Test command timed out: {executable} {string.Join(" ", arguments)}"); + } + Task.WaitAll(stdout, stderr); + process.ExitCode.Should().Be(0, stderr.Result); + return stdout.Result.Trim(); + } + } +} diff --git a/.ci/unit_tests/Python_Engine_Test/Compute/Pytest.cs b/.ci/unit_tests/Python_Engine_Test/Compute/Pytest.cs new file mode 100644 index 00000000..36157922 --- /dev/null +++ b/.ci/unit_tests/Python_Engine_Test/Compute/Pytest.cs @@ -0,0 +1,101 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using FluentAssertions; +using BH.oM.Base.Debugging; +using System.Text.RegularExpressions; +using EngineCompute = BH.Engine.Python.Compute; +using EngineQuery = BH.Engine.Python.Query; + +namespace BH.Tests.Engine.Python.Compute +{ + [NonParallelizable] + public class Pytest + { + [Test, Order(9999)] + public void Execute_Pytests() + { + DirectoryInfo? repository = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (repository != null && !File.Exists(Path.Combine(repository.FullName, "Python_Engine", "Python", "pyproject.toml"))) + repository = repository.Parent; + repository.Should().NotBeNull("the toolkit sources must be available to run pytest"); + + string source = Path.Combine(repository!.FullName, "Python_Engine", "Python"); + string name = "bhom_pytest_" + Guid.NewGuid().ToString("N"); + string directory = Path.Combine(EngineQuery.DirectoryEnvironments(), name); + BH.Engine.Base.Compute.ClearCurrentEvents(); + + try + { + Directory.CreateDirectory(directory); + foreach (string file in new[] { "pyproject.toml", "README.md" }) + File.Copy(Path.Combine(source, file), Path.Combine(directory, file)); + foreach (string folder in new[] { "src", "tests" }) + { + string sourceFolder = Path.Combine(source, folder); + foreach (string file in Directory.EnumerateFiles(sourceFolder, "*", SearchOption.AllDirectories)) + { + string relative = Path.GetRelativePath(sourceFolder, file); + if (relative.Split(Path.DirectorySeparatorChar).Contains("__pycache__")) + continue; + string destination = Path.Combine(directory, folder, relative); + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + File.Copy(file, destination); + } + } + + var venv = EngineCompute.VirtualEnvironment(Path.Combine(directory, "pyproject.toml"), reload: false); + venv.Should().NotBeNull("environment installation must succeed: {0}", + string.Join(Environment.NewLine, BH.Engine.Base.Query.CurrentEvents().Where(e => e.Type == EventType.Error).Select(e => e.Message))); + + // Sync explicitly: a reloaded environment is not guaranteed to include dev dependencies. + var sync = EngineCompute.RunCommandStdout( + $"\"{EngineQuery.UVExecutable()}\" sync --all-packages --group dev", startDirectory: directory); + sync.Item3.Should().Be(0, "dev dependency installation must succeed:\n{0}\n{1}", sync.Item1, sync.Item2); + + var result = EngineCompute.RunCommandStdout( + $"set \"SKIP_LONG_PYTEST=1\" && \"{venv.Executable}\" -m pytest -ra", + startDirectory: directory, timeoutMinutes: 10); + string output = result.Item1 + Environment.NewLine + result.Item2; + TestContext.Progress.WriteLine(output); + + Match warnings = Regex.Match(output, + @"(?ms)^=+[ \t]*warnings summary[ \t]*=+\r?\n(?.*?)(?=^=+|\z)"); + if (warnings.Success) + BH.Engine.Base.Compute.RecordWarning(warnings.Groups["warnings"].Value.Trim()); + if (result.Item3 != 0) + BH.Engine.Base.Compute.RecordError($"Pytest failed (exit code {result.Item3}).\n{output}"); + + result.Item3.Should().Be(0, "pytest must pass:\n{0}", output); + BH.Engine.Base.Query.CurrentEvents().Where(e => e.Type == EventType.Error) + .Select(e => e.Message).Should().BeEmpty(); + } + finally + { + if (EngineQuery.VirtualEnvironmentExists(name)) + EngineCompute.RemoveVirtualEnvironment(name, run: true); + if (Directory.Exists(directory)) + Directory.Delete(directory, true); + } + } + } +} \ No newline at end of file diff --git a/.ci/unit_tests/Python_Engine_Test/Compute/Remove.cs b/.ci/unit_tests/Python_Engine_Test/Compute/Remove.cs new file mode 100644 index 00000000..74f2ac23 --- /dev/null +++ b/.ci/unit_tests/Python_Engine_Test/Compute/Remove.cs @@ -0,0 +1,130 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using FluentAssertions; +using EngineCompute = BH.Engine.Python.Compute; +using EngineQuery = BH.Engine.Python.Query; + +namespace BH.Tests.Engine.Python.Compute +{ + [NonParallelizable] + public class RemoveTests : InstallRemoveTestBase + { + [Test] + public void RemoveKernel_WarnsWhenEnvironmentDoesNotExist() + { + EngineCompute.RemoveJupyterKernel(m_Environment.Name + "_missing", run: true); + + AssertWarningWithoutErrors(); + } + + [Test] + public void RemoveKernel_WarnsWhenKernelDoesNotExist() + { + EngineCompute.RemoveJupyterKernel(m_Environment.Name, run: true); + + AssertWarningWithoutErrors(); + } + + [Test] + public void RemoveKernel_RemovesExistingKernelThenWarnsOnRepeat() + { + EngineCompute.InstallJupyterKernel(m_Environment); + AssertNoErrors(); + BH.Engine.Base.Compute.ClearCurrentEvents(); + + EngineCompute.RemoveJupyterKernel(m_Environment.Name, run: true); + + Directory.Exists(Path.Combine(m_Root, "jupyter", "kernels", m_Environment.Name)).Should().BeFalse(); + AssertNoErrors(); + EngineCompute.RemoveJupyterKernel(m_Environment.Name, run: true); + AssertWarningWithoutErrors(); + } + + [Test] + public void RemoveVirtualEnvironment_WarnsWhenEnvironmentDoesNotExist() + { + EngineCompute.RemoveVirtualEnvironment(m_Environment.Name + "_missing", run: true); + + AssertWarningWithoutErrors(); + } + + [Test] + public void RemoveVirtualEnvironment_RemovesEnvironmentWithoutKernelThenWarnsOnRepeat() + { + EngineCompute.RemoveVirtualEnvironment(m_Environment.Name, run: true); + + Directory.Exists(m_Root).Should().BeFalse(); + AssertWarningWithoutErrors(); + BH.Engine.Base.Compute.ClearCurrentEvents(); + EngineCompute.RemoveVirtualEnvironment(m_Environment.Name, run: true); + AssertWarningWithoutErrors(); + } + + [Test] + public void RemoveUV_WarnsWhenManagedUVDoesNotExist() + { + CheckUVRemoval(managedUVExists: false); + } + + [Test] + public void RemoveUV_RemovesManagedToolsAndIgnoresExternalUV() + { + CheckUVRemoval(managedUVExists: true); + } + + private void CheckUVRemoval(bool managedUVExists) + { + string toolsDirectory = EngineQuery.DirectoryTools(); + string externalDirectory = Path.Combine(m_Root, "external-uv"); + Directory.CreateDirectory(externalDirectory); + string externalUV = Path.Combine(externalDirectory, "uv.exe"); + File.Copy(EngineQuery.UVExecutable(), externalUV); + byte[] externalBytes = File.ReadAllBytes(externalUV); + SetVariable("PATH", externalDirectory + Path.PathSeparator + Environment.GetEnvironmentVariable("PATH")); + + WithIsolatedDirectory(toolsDirectory, () => + { + string marker = Path.Combine(toolsDirectory, "other-tool.txt"); + File.WriteAllText(marker, "preserve when UV is missing"); + if (managedUVExists) + File.Copy(externalUV, EngineQuery.UVExecutable()); + + EngineCompute.RemoveUV(run: true); + + if (managedUVExists) + { + Directory.Exists(toolsDirectory).Should().BeFalse(); + AssertNoErrors(); + BH.Engine.Base.Query.CurrentEvents().Should().BeEmpty(); + } + else + { + File.Exists(marker).Should().BeTrue(); + AssertWarningWithoutErrors(); + } + File.ReadAllBytes(externalUV).Should().Equal(externalBytes); + Run(externalUV, "--version").Should().StartWith("uv "); + }); + } + } +} diff --git a/.ci/unit_tests/Python_Engine_Test/Compute/Run.cs b/.ci/unit_tests/Python_Engine_Test/Compute/Run.cs new file mode 100644 index 00000000..ec2544e4 --- /dev/null +++ b/.ci/unit_tests/Python_Engine_Test/Compute/Run.cs @@ -0,0 +1,89 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using BH.oM.Base.Debugging; +using FluentAssertions; +using System.Diagnostics; +using EngineCompute = BH.Engine.Python.Compute; + +namespace BH.Tests.Engine.Python.Compute +{ + [NonParallelizable] + public class RunTests + { + [SetUp] + [TearDown] + public void CleanUp() + { + BH.Engine.Base.Compute.ClearCurrentEvents(); + } + + [TestCase(0)] + [TestCase(7)] + public void RunCommandStdout_ReturnsBothStreamsAndExitCode(int exitCode) + { + SynchronizationContext? original = SynchronizationContext.Current; + try + { + SynchronizationContext.SetSynchronizationContext(new NonPumpingContext()); + var result = EngineCompute.RunCommandStdout( + $"(for /L %i in (1,1,2000) do @(echo output& echo diagnostic 1>&2)) & exit /b {exitCode}"); + + result.Should().NotBeNull("failed commands must still return both streams and their exit code"); + result.Item1.Split('\n').Should().HaveCount(2000); + result.Item2.Split('\n').Should().HaveCount(2000); + result.Item1.Should().StartWith("output").And.EndWith("output"); + result.Item2.Should().StartWith("diagnostic").And.EndWith("diagnostic"); + result.Item3.Should().Be(exitCode); + BH.Engine.Base.Query.CurrentEvents().Any(e => e.Type == EventType.Error).Should().Be(exitCode != 0); + } + finally + { + SynchronizationContext.SetSynchronizationContext(original); + } + } + + [Test] + public void RunCommandStdout_TimeoutRetainsBothStreams() + { + Stopwatch timer = Stopwatch.StartNew(); + var result = EngineCompute.RunCommandStdout( + "echo started & echo diagnostic 1>&2 & powershell.exe -NoProfile -NonInteractive -Command \"Start-Sleep -Seconds 30\"", + timeoutMinutes: 0.02); + + result.Should().NotBeNull("timeouts must still return captured streams and exit code -1"); + result.Item1.Should().Be("started"); + result.Item2.Should().Be("diagnostic"); + result.Item3.Should().Be(-1); + timer.Elapsed.Should().BeLessThan(TimeSpan.FromSeconds(10)); + BH.Engine.Base.Query.CurrentEvents().Should().Contain(e => e.Type == EventType.Error && e.Message.Contains("timeout")); + } + + private sealed class NonPumpingContext : SynchronizationContext + { + public override void Post(SendOrPostCallback callback, object? state) + { + throw new InvalidOperationException("The command runner captured the host synchronization context."); + } + } + } +} diff --git a/.ci/unit_tests/Python_Engine_Test/Compute/VirtualEnvironment.cs b/.ci/unit_tests/Python_Engine_Test/Compute/VirtualEnvironment.cs new file mode 100644 index 00000000..7c921ad2 --- /dev/null +++ b/.ci/unit_tests/Python_Engine_Test/Compute/VirtualEnvironment.cs @@ -0,0 +1,139 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using BH.oM.Base.Debugging; +using BH.oM.Python; +using FluentAssertions; +using EngineCompute = BH.Engine.Python.Compute; +using EngineQuery = BH.Engine.Python.Query; + +namespace BH.Tests.Engine.Python.Compute +{ + [NonParallelizable] + public class VirtualEnvironmentTests + { + private readonly Dictionary m_OriginalVariables = []; + private string m_TestDirectory = null!; + private string m_Name = null!; + private string m_Directory = null!; + + [OneTimeSetUp] + public void SetUpTools() + { + m_TestDirectory = Path.Combine(Path.GetTempPath(), "bhom_lifecycle_" + Guid.NewGuid().ToString("N")); + foreach (string variable in new[] { "UV_TOOL_DIR", "UV_TOOL_BIN_DIR", "JUPYTER_DATA_DIR", "JUPYTER_PATH" }) + { + m_OriginalVariables[variable] = Environment.GetEnvironmentVariable(variable); + Environment.SetEnvironmentVariable(variable, Path.Combine(m_TestDirectory, variable)); + } + } + + [SetUp] + public void SetUp() + { + m_Name = "bhom_lifecycle_" + Guid.NewGuid().ToString("N"); + m_Directory = Path.Combine(EngineQuery.DirectoryEnvironments(), m_Name); + BH.Engine.Base.Compute.ClearCurrentEvents(); + } + + [TearDown] + public void CleanUpEnvironment() + { + if (EngineQuery.VirtualEnvironmentExists(m_Name)) + EngineCompute.RemoveVirtualEnvironment(m_Name, run: true); + if (Directory.Exists(m_Directory)) + Directory.Delete(m_Directory, true); + BH.Engine.Base.Compute.ClearCurrentEvents(); + } + + [OneTimeTearDown] + public void CleanUpTools() + { + foreach (var variable in m_OriginalVariables) + Environment.SetEnvironmentVariable(variable.Key, variable.Value); + if (Directory.Exists(m_TestDirectory)) + Directory.Delete(m_TestDirectory, true); + if (Directory.Exists(EngineQuery.DirectoryTools())) + Directory.Delete(EngineQuery.DirectoryTools(), true); + } + + [Test] + public void FromTargetVersion_CreatesReloadsAndRemovesEnvironmentAndKernel() + { + PythonEnvironment venv = EngineCompute.VirtualEnvironment("3.11", m_Name); + venv.Should().NotBeNull(); + File.Exists(venv.Executable).Should().BeTrue(); + EngineQuery.Version(venv).Should().StartWith("3.11."); + EngineCompute.RunPythonString(venv, "import ipykernel; print('ready')").Should().Be("ready"); + EngineCompute.VirtualEnvironment("3.11", m_Name, reload: true).Executable.Should().Be(venv.Executable); + File.Exists(KernelFile()).Should().BeTrue(); + + EngineCompute.RemoveVirtualEnvironment(m_Name, run: true); + + Directory.Exists(m_Directory).Should().BeFalse(); + File.Exists(KernelFile()).Should().BeFalse(); + AssertNoErrors(); + } + + [Test] + public void FromTOML_InstallsBHoMVersionRecreatesAndRemovesEnvironment() + { + string packageDirectory = Path.Combine(m_Directory, "src", m_Name); + Directory.CreateDirectory(packageDirectory); + File.WriteAllText(Path.Combine(packageDirectory, "__init__.py"), ""); + File.WriteAllText(Path.Combine(m_Directory, "README.md"), ""); + string tomlFile = Path.Combine(m_Directory, "pyproject.toml"); + File.WriteAllText(tomlFile, + $"[project]\nname = '{m_Name}'\nversion = '0.1.0' # project version\nreadme = 'README.md'\n" + + "requires-python = '>=3.11,<3.12'\ndependencies = []\n" + + "[build-system]\nrequires = ['hatchling']\nbuild-backend = 'hatchling.build'\n" + + "[tool.bhom]\nversion = 'leave-this-alone'\n"); + + PythonEnvironment venv = EngineCompute.VirtualEnvironment(tomlFile, reload: false); + venv.Should().NotBeNull(); + string version = BH.Engine.Base.Query.BHoMVersion(); + File.ReadAllText(tomlFile).Should().Contain($"version = \"{version}\" # project version") + .And.Contain("version = 'leave-this-alone'"); + EngineCompute.RunPythonString(venv, + $"import importlib.metadata; print(importlib.metadata.version('{m_Name}'))").Should().Be(version); + + EngineCompute.VirtualEnvironment(tomlFile, reload: false).Should().NotBeNull(); + File.Exists(Path.Combine(packageDirectory, "__init__.py")).Should().BeTrue(); + EngineCompute.VirtualEnvironment(tomlFile, reload: true).Executable.Should().Be(venv.Executable); + EngineCompute.RemoveVirtualEnvironment(m_Name, run: true); + + Directory.Exists(m_Directory).Should().BeFalse(); + File.Exists(KernelFile()).Should().BeFalse(); + AssertNoErrors(); + } + + private string KernelFile() + { + return Path.Combine(Environment.GetEnvironmentVariable("JUPYTER_DATA_DIR")!, "kernels", m_Name, "kernel.json"); + } + + private static void AssertNoErrors() + { + BH.Engine.Base.Query.CurrentEvents().Where(e => e.Type == EventType.Error).Select(e => e.Message).Should().BeEmpty(); + } + } +} diff --git a/.ci/unit_tests/Python_Engine_Test/Python_Engine_Test.csproj b/.ci/unit_tests/Python_Engine_Test/Python_Engine_Test.csproj new file mode 100644 index 00000000..1a39960b --- /dev/null +++ b/.ci/unit_tests/Python_Engine_Test/Python_Engine_Test.csproj @@ -0,0 +1,41 @@ + + + + net9.0 + latest + enable + enable + false + BH.Tests.Engine.Python + + + + + + + + + + + + + + + + + + + ..\..\..\..\..\..\..\..\..\ProgramData\BHoM\Assemblies\BHoM.dll + True + + + ..\..\..\..\..\..\..\..\..\ProgramData\BHoM\Assemblies\BHoM_Engine.dll + True + + + + + + + + diff --git a/.ci/unit_tests/Python_Engine_Test/Query/Directory.cs b/.ci/unit_tests/Python_Engine_Test/Query/Directory.cs new file mode 100644 index 00000000..bcca18c8 --- /dev/null +++ b/.ci/unit_tests/Python_Engine_Test/Query/Directory.cs @@ -0,0 +1,51 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using FluentAssertions; + +namespace BH.Tests.Engine.Python.Query +{ + public class Directory_Tests + { + [Test] + public void DirectoryExtension() + { + string extensionsDirectory = "C:\\ProgramData\\BHoM\\Extensions"; + var result = new DirectoryInfo(BH.Engine.Python.Query.DirectoryExtensions()); + + Assert.That(result, Is.Not.Null); + result.FullName.Should().Be(extensionsDirectory); + Directory.Exists(extensionsDirectory).Should().BeTrue(); + } + + [Test] + public void DirectoryEnvironments() + { + string environmentsDirectory = "C:\\ProgramData\\BHoM\\Extensions\\Python"; + var result = new DirectoryInfo(BH.Engine.Python.Query.DirectoryEnvironments()); + + Assert.That(result, Is.Not.Null); + result.FullName.Should().Be(environmentsDirectory); + Directory.Exists(environmentsDirectory).Should().BeTrue(); + } + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 48d3a171..2ce1d0e1 100644 --- a/.gitignore +++ b/.gitignore @@ -147,7 +147,9 @@ instance/ .scrapy # Sphinx documentation -docs/_build/ +**/docs/_build/ +**/docs/index.rst +**/docs/autoapi # PyBuilder .pybuilder/ @@ -216,5 +218,8 @@ dmypy.json # Cython debug symbols cython_debug/ -# unignored files -!example.sql +# Additional directories to ignore +.benchmarks/ + +# UV +uv.lock diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs deleted file mode 100644 index 8ec03656..00000000 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ /dev/null @@ -1,86 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using BH.oM.Base.Attributes; -using BH.oM.Python; -using BH.oM.Python.Enums; -using System.Collections.Generic; -using System.ComponentModel; -using System.Diagnostics; -using System.IO; -using System.Linq; - -namespace BH.Engine.Python -{ - public static partial class Compute - { - [Description("Retrieve or reinstall the base Python Environment for BHoM workflows.")] - [Input("version", "The target version of python to be installed or retrieved.")] - [Input("reload", "Reload the base Python environment rather than recreating it, if it already exists.")] - [Input("run", "Start the installation/retrieval of the BHoM Base Python Environment.")] - [Output("env", "The base Python Environment for all BHoM workflows.")] - public static PythonEnvironment BasePythonEnvironment( - PythonVersion version = PythonVersion.v3_10, - bool reload = true, - bool run = false - ) - { - if (!run) - { - return null; - } - - if (!Directory.Exists(Query.DirectoryEnvironments())) - { - // create PythonEnvironments directory if it doesnt already exist - Directory.CreateDirectory(Query.DirectoryEnvironments()); - } - - // determine whether the base environment already exists - string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe"); - bool exists = File.Exists(targetExecutable); - - if (exists && reload) - { - PythonEnvironment env = new PythonEnvironment() { Name = Query.ToolkitName(), Executable = targetExecutable }; - UpdateBHoMPackages(env); - return env; - } - - if (exists && !reload) - // remove all existing environments and kernels - RemoveEverything(); - - // download and run the installer for the target Python version - string exe = version.DownloadPythonVersion(); - - // install essential packages into base environment - InstallPackages(exe, new List() { "virtualenv", "jupyterlab", "black", "pylint" }); - InstallPackageLocal(exe, Path.Combine(Query.DirectoryCode(), Query.ToolkitName())); - - return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = exe }; - } - } -} - - - diff --git a/Python_Engine/Compute/Download.cs b/Python_Engine/Compute/Download.cs deleted file mode 100644 index 054f03aa..00000000 --- a/Python_Engine/Compute/Download.cs +++ /dev/null @@ -1,152 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using BH.oM.Base.Attributes; -using BH.oM.Python.Enums; -using System; -using System.ComponentModel; -using System.Diagnostics; -using System.IO; -using System.Reflection; -using System.Xml.Linq; - -namespace BH.Engine.Python -{ - public static partial class Compute - { - [Description("Download a file from a Url.")] - [Input("fileUrl", "The file to download.")] - [Input("targetDirectory", "The destination directory to download the file to.")] - [Input("filename", "An optional input to rename the downloaded file.")] - [Input("overwrite", "An optional input to overwrite the file if it already exists, otherwise if it does then return the existing file.")] - [Output("filePath", "The path to the downloaded file.")] - public static string DownloadFile( - string fileUrl, - string targetDirectory, - string filename = null, - bool overwrite = true - ) - { - if (string.IsNullOrWhiteSpace(fileUrl)) - { - BH.Engine.Base.Compute.RecordError($"The fileUrl cannot be null or empty."); - return null; - } - - - if (string.IsNullOrWhiteSpace(targetDirectory)) - { - BH.Engine.Base.Compute.RecordError($"The targetDirectory cannot be null or empty."); - return null; - } - - - if (!Directory.Exists(targetDirectory)) - { - BH.Engine.Base.Compute.RecordError($"The targetDirectory \"{targetDirectory}\" does not exist."); - return null; - } - - - if (string.IsNullOrWhiteSpace(filename)) - filename = Path.GetFileName(fileUrl); - - string filePath = Path.Combine(targetDirectory, filename); - - if (File.Exists(filePath)) - { - if (!overwrite) - return filePath; - File.Delete(filePath); - } - - using (var client = new System.Net.WebClient()) - { - client.DownloadFile(fileUrl, filePath); - } - - return filePath; - } - - /******************************************************/ - - [Description("Download and install a specified version of python, and return the executable for it.")] - [Input("version", "The version of python to download.")] - [Output("pythonExecutable", "The executable (python.exe) for the python version that was installed.")] - public static string DownloadPythonVersion(this PythonVersion version) - { - string url = version.EmbeddableURL(); - - string basePath = Path.Combine(Query.DirectoryBaseEnvironment(version)); - - if (File.Exists(Path.Combine(basePath, "python.exe"))) - return Path.Combine(basePath, "python.exe"); - - if (!Directory.Exists(basePath)) - Directory.CreateDirectory(basePath); - else - { - Directory.Delete(basePath, true); //if there are any files here already for some reason, remove them. - Directory.CreateDirectory(basePath); - } - - string installerFile = DownloadFile(url, basePath, "installer.exe"); - - using (Process install = new Process() - { - StartInfo = new ProcessStartInfo() - { - FileName = installerFile, - Arguments = $"/passive InstallAllUsers=0 InstallLauncherAllUsers=0 Include_launcher=0 Shortcuts=0 AssociateFiles=0 Include_tools=0 Include_test=0 TargetDir={Modify.AddQuotesIfRequired(basePath)}", - RedirectStandardError = true, - UseShellExecute = false, - } - }) - { - install.StartInfo.Environment["PYTHONHOME"] = ""; - install.Start(); - string stderr = install.StandardError.ReadToEnd(); - install.WaitForExit(); - if (install.ExitCode != 0) - { - BH.Engine.Base.Compute.RecordError($"Error installing python: {stderr}"); - return null; - } - } - - string pythonExePath = Path.Combine(basePath, "python.exe"); - - // It is possible for the installer to exit with ExitCode==0 without installing Python in the requested location. This can happen if the same version of Python appears to be installed elsewhere on the system. It can also happen if there was a 'dodgy' uninstall of a previous version. We've been unable to figure out where the installer is actually looking for existing installs. To fix dodgy uninstalls, run the installer again for the target version of Python (download links within EmbeddableURL.cs) and run a combination of 'repair' and 'uninstall' until it's gone. Installing the py launcher (via 'modify' in the installer) before attempting to uninstall could also help. - - if (!File.Exists(pythonExePath)) - { - BH.Engine.Base.Compute.RecordError($"The Python installer appears to have completed successfully, but \n{pythonExePath} does not exist. \nThis usually means that Python {version} already exists on your machine, but in a different location. \nThis toolkit is therefore unable to run any Python commands right now. \nTry uninstalling Python {version} from your system before running this BHoM method again."); - return null; - } - - return pythonExePath; - } - } -} - - - diff --git a/Python_Engine/Compute/Install.cs b/Python_Engine/Compute/Install.cs index 7b7f0c91..81520045 100644 --- a/Python_Engine/Compute/Install.cs +++ b/Python_Engine/Compute/Install.cs @@ -25,160 +25,189 @@ using System; using System.Collections.Generic; using System.ComponentModel; -using System.Diagnostics; using System.IO; -using System.Xml.Linq; +using System.Linq; namespace BH.Engine.Python { public static partial class Compute { - [Description("Install a list of packages to a given Python executable.")] - [Input("executable", "The Python executable to install the packages to.")] - [Input("packages", "The packages to install.")] - public static void InstallPackages( - string executable, - List packages - ) + [Description("Install UV in the BHoM tools directory and add that directory to the process PATH. Install any missing ruff, jupyterlab and sphinx tools; UV installations elsewhere are not used.")] + [Input("run", "Set to True to install UV and its core tools.")] + public static void InstallUV(bool run = false) { - if (!File.Exists(executable)) - { - BH.Engine.Base.Compute.RecordError("Python executable not found at " + executable); - } + if (!run) + return; - if (packages.Count == 0) - { - BH.Engine.Base.Compute.RecordError("No packages to install."); - } + string toolsDirectory = Query.DirectoryTools(); - string packagesString = string.Join(" ", packages); - System.Diagnostics.Process process = new System.Diagnostics.Process() + string uvExecutable = Query.UVExecutable(); + if (!File.Exists(uvExecutable)) { - StartInfo = new System.Diagnostics.ProcessStartInfo() + var installation = RunCommandStdout( + command: $"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command \"$ErrorActionPreference = 'Stop'; $env:UV_INSTALL_DIR = '{toolsDirectory.Replace("'", "''")}'; $env:UV_NO_MODIFY_PATH = '1'; irm https://astral.sh/uv/install.ps1 | iex\""); + if (installation.Item3 != 0) + return; + if (!File.Exists(uvExecutable)) { - FileName = executable, - Arguments = $"-m pip install --no-warn-script-location {packagesString}", - UseShellExecute = false, - RedirectStandardError = true + Base.Compute.RecordError($"UV installation did not produce an executable at {uvExecutable}."); + return; } - }; - process.StartInfo.Environment["PYTHONHOME"] = ""; - using (Process p = Process.Start(process.StartInfo)) + } + + string[] otherPaths = (Environment.GetEnvironmentVariable("PATH") ?? string.Empty).Split(Path.PathSeparator) + .Where(directory => !string.Equals(directory.Trim().Trim('"'), toolsDirectory, StringComparison.OrdinalIgnoreCase)) + .ToArray(); + Environment.SetEnvironmentVariable("PATH", string.Join(Path.PathSeparator.ToString(), new[] { toolsDirectory }.Concat(otherPaths))); + + var installedTools = RunCommandStdout($"\"{uvExecutable}\" tool list"); + if (installedTools.Item3 != 0) + return; + + var installedNames = new HashSet( + installedTools.Item1.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) + .Where(line => !line.StartsWith("-")) + .Select(line => line.Split(' ')[0]), StringComparer.OrdinalIgnoreCase); + foreach (string tool in new[] { "ruff", "jupyterlab", "sphinx" }) { - string standardError = p.StandardError.ReadToEnd(); - p.WaitForExit(); - if (p.ExitCode != 0) - BH.Engine.Base.Compute.RecordError($"Error installing packages [{packagesString}].\n{standardError}"); + if (!installedNames.Contains(tool)) + RunCommandStdout($"\"{uvExecutable}\" tool install {tool}"); } } - [Description("Install a list of packages to a given Python executable.")] - [Input("environment", "The Python environment to install the packages to.")] - [Input("packages", "The packages to install.")] - public static void InstallPackages( - this PythonEnvironment environment, - List packages - ) + [Description("Register a Jupyter kernel for a given venv unless that kernel already exists.")] + [Input("venv", "The Python environment to install the packages to.")] + public static void InstallJupyterKernel(this PythonEnvironment venv) { - InstallPackages(environment.Executable, packages); + if (JupyterKernelExists(venv, venv.Name, true) != false) + return; + + venv.InstallPackages(packages: new List() { "ipykernel" }, true); + string directory = Query.VirtualEnvironmentDirectory(venv.Name); + + string cmd = $"\"{venv.Executable}\" -m ipykernel install --user --name \"{venv.Name}\" --display-name \"{venv.Name}\""; + RunCommandStdout(command: cmd, startDirectory: directory); } - [Description("Install packages from a requirements.txt file to a given Python executable.")] - [Input("executable", "The Python executable to install the packages to.")] - [Input("requirements", "The requirements.txt file to install.")] - public static void InstallRequirements( - string executable, - string requirements + [PreviousVersion("9.3", "BH.Engine.Python.Compute.InstallPackages(PythonEnvironment, List)")] + [Description("Install a list of packages to a given Python executable. UV retains satisfied versions and registers project dependencies when applicable.")] + [Input("venv", "The Python environment to install the packages to.")] + [Input("packages", "The packages to install.")] + [Input("run", "Run this method.")] + public static void InstallPackages( + this PythonEnvironment venv, + List packages, + bool run = false ) { - if (!File.Exists(executable)) - { - BH.Engine.Base.Compute.RecordError($"Python executable not found at {executable}"); - } + string directory = Query.VirtualEnvironmentDirectory(venv.Name); + bool isProject = File.Exists(Path.Combine(directory, "pyproject.toml")); - if (!File.Exists(requirements)) - { - BH.Engine.Base.Compute.RecordError($"requirements.txt not found at {requirements}"); - } + string uvCommand = isProject ? "add" : "pip install"; + string cmd = $"\"{Query.UVExecutable()}\" {uvCommand} --python \"{venv.Executable}\" {string.Join(" ", packages.Select(package => $"\"{package}\""))}"; - - System.Diagnostics.Process process = new System.Diagnostics.Process() + if (run) { - StartInfo = new System.Diagnostics.ProcessStartInfo() - { - FileName = executable, - Arguments = $"-m pip install --no-warn-script-location -r {Modify.AddQuotesIfRequired(requirements)}", - UseShellExecute = false, - RedirectStandardError = true, - } - }; - process.StartInfo.Environment["PYTHONHOME"] = ""; - using (Process p = Process.Start(process.StartInfo)) + RunCommandStdout(command: cmd, startDirectory: directory); + } + else { - string standardError = p.StandardError.ReadToEnd(); - p.WaitForExit(); - if (p.ExitCode != 0) - BH.Engine.Base.Compute.RecordError($"Error installing packages from {requirements}.\n{standardError}"); + Base.Compute.RecordNote($"The following command will be executed from \"{directory}\": {cmd}"); } } - [Description("Install packages from a requirements.txt file to a given Python environment.")] - [Input("environment", "The Python environment to install the packages to.")] - [Input("requirements", "The requirements.txt file to install.")] - public static void InstallRequirements( - this PythonEnvironment environment, - string requirements - ) - { - InstallRequirements(environment.Executable, requirements); - } - - - [Description("Install a local Python package to a given Python executable.")] - [Input("executable", "The Python executable to install the package to.")] + [PreviousVersion("9.3", "BH.Engine.Python.Compute.InstallPackages(PythonEnvironment, string)")] + [Description("Install a local editable Python package. An unchanged installation from the same directory is retained; project dependencies are still registered.")] + [Input("venv", "The Python environment to install the packages to.")] [Input("packageDirectory", "The package to install.")] + [Input("run", "Run this method.")] public static void InstallPackageLocal( - string executable, - string packageDirectory + this PythonEnvironment venv, + string packageDirectory, + bool run = false ) { - if (!File.Exists(executable)) + if (!Directory.Exists(packageDirectory)) { - BH.Engine.Base.Compute.RecordError("Python executable not found at " + executable); + Base.Compute.RecordError($"{packageDirectory} not found."); + return; } - - System.Diagnostics.Process process = new System.Diagnostics.Process() + + packageDirectory = Path.GetFullPath(packageDirectory); + string directory = Query.VirtualEnvironmentDirectory(venv.Name); + bool isProject = File.Exists(Path.Combine(directory, "pyproject.toml")); + + string uvCommand = isProject ? "add" : "pip install"; + string cmd = $"\"{Query.UVExecutable()}\" {uvCommand} --python \"{venv.Executable}\" --editable \"{packageDirectory}\""; + + if (run) { - StartInfo = new System.Diagnostics.ProcessStartInfo() + if (!isProject) { - FileName = executable, - Arguments = $"-m pip install --no-warn-script-location -e {Modify.AddQuotesIfRequired(packageDirectory)}", - UseShellExecute = false, - RedirectStandardError = true, + // Explicit local paths are otherwise always reinstalled by UV. + string script = "import importlib.metadata as m,json,pathlib,sys; " + + "p=pathlib.Path(sys.argv[1]).resolve(); " + + "sources=[p/f for f in ('pyproject.toml','setup.py','setup.cfg') if (p/f).exists()]; " + + "print(any((lambda u: u.get('dir_info',{}).get('editable',False) and " + + "u.get('url','').rstrip('/').lower()==p.as_uri().rstrip('/').lower() and " + + "all(f.stat().st_mtime<=pathlib.Path(d.locate_file(x)).stat().st_mtime for f in sources))" + + "(json.loads(d.read_text('direct_url.json') or '{}')) " + + "for d in m.distributions() for x in (d.files or []) if str(x).endswith('.dist-info/METADATA')))"; + var installed = RunCommandStdout( + $"\"{venv.Executable}\" -c \"{script}\" \"{packageDirectory}\"", startDirectory: directory); + if (installed.Item3 != 0) + return; + if (installed.Item1 == "True") + return; + if (installed.Item1 != "False") + { + Base.Compute.RecordError($"Could not check the local package installation in {venv.Name}."); + return; + } } - }; - process.StartInfo.Environment["PYTHONHOME"] = ""; - using (Process p = Process.Start(process.StartInfo)) + RunCommandStdout(command: cmd, startDirectory: directory); + } + else { - string standardError = p.StandardError.ReadToEnd(); - p.WaitForExit(); - if (p.ExitCode != 0) - BH.Engine.Base.Compute.RecordError($"Error installing package from \"{packageDirectory}\".\n{standardError}"); + Base.Compute.RecordNote($"The following command will be executed from \"{directory}\": {cmd}"); } } - [Description("Install a local Python package to a given Python environment.")] - [Input("environment", "The Python environment to install the packages to.")] - [Input("packageDirectory", "The package to install.")] - public static void InstallPackageLocal( - this PythonEnvironment environment, - string packageDirectory + private static bool? JupyterKernelExists(PythonEnvironment venv, string name, bool requireMatchingExecutable = false) + { + string script = "import importlib.util,pathlib,sys; " + + "available=importlib.util.find_spec('jupyter_client') is not None; " + + "manager=__import__('jupyter_client.kernelspec',fromlist=['KernelSpecManager']).KernelSpecManager() if available else None; " + + "exists=available and sys.argv[1].lower() in manager.find_kernel_specs(); " + + "print(exists and (sys.argv[2]=='False' or " + + "pathlib.Path(manager.get_kernel_spec(sys.argv[1].lower()).argv[0]).resolve()==pathlib.Path(sys.executable).resolve()))"; + var result = RunCommandStdout($"\"{venv.Executable}\" -c \"{script}\" \"{name}\" {requireMatchingExecutable}"); + if (result.Item3 != 0) + return null; + if (result.Item1 == "True") + return true; + if (result.Item1 == "False") + return false; + + Base.Compute.RecordError($"Could not check whether Jupyter kernel {name} exists."); + return null; + } + + [Description("Install the Python_Toolkit Python venv.")] + [Input("run", "Run this method.")] + [Input("reload", "Reload the virtual environment rather than recreating it, if it already exists.")] + [Output("venv", "The resultant venv.")] + public static PythonEnvironment InstallPythonToolkitVenv( + bool run = false, + bool reload = true ) { - InstallPackageLocal(environment.Executable, packageDirectory); + string tomlFile = Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName(), "pyproject.toml"); + if (run) + { + return VirtualEnvironment(tomlFile: tomlFile, reload: reload); + } + return null; } } } - - - diff --git a/Python_Engine/Compute/Remove.cs b/Python_Engine/Compute/Remove.cs index 46adb528..77f4b094 100644 --- a/Python_Engine/Compute/Remove.cs +++ b/Python_Engine/Compute/Remove.cs @@ -21,123 +21,115 @@ */ using BH.oM.Base.Attributes; -using BH.oM.Python.Enums; -using System; -using System.Collections.Generic; +using BH.oM.Python; using System.ComponentModel; -using System.Diagnostics; using System.IO; namespace BH.Engine.Python { public static partial class Compute { - - [Description("Remove a named BHoM Python kernel.")] - [Input("kernelName", "The name of the kernel to remove.")] - public static void RemoveKernel(string kernelName) + [Description("Remove the named Python virtual environment. Missing environments produce a warning instead of an error.")] + [Input("name", "The name of the Python virtual environment to remove.")] + [Input("run", "Run this method.")] + public static void RemoveVirtualEnvironment(string name, bool run = false) { - string kernelPath = Path.Combine(Query.DirectoryKernels(), kernelName.ToLower()); - if (Directory.Exists(kernelPath)) - Directory.Delete(kernelPath, true); - } - - [Description("Remove all BHoM Python kernels.")] - public static void RemoveAllKernels() - { - DirectoryInfo di = new DirectoryInfo(Query.DirectoryKernels()); - foreach (FileInfo file in di.EnumerateFiles()) + if (!Query.VirtualEnvironmentExists(name)) { - file.Delete(); + Base.Compute.RecordWarning($"No Python environment named {name} exists."); + return; } - foreach (DirectoryInfo dir in di.EnumerateDirectories()) + + string venvDirectory = Path.Combine(Query.DirectoryEnvironments(), name); + + if (run) { - dir.Delete(true); + RemoveJupyterKernel(name, run); + + try + { + Directory.Delete(venvDirectory, true); + } + catch (IOException ex) + { + Base.Compute.RecordError($"Failed to remove virtual environment directory {venvDirectory}: {ex.Message}"); + } } } - [Description("Remove the named BHoM Python virtual environment.")] - [Input("envName", "The name of the BHoM Python virtual environment to remove.")] - public static void RemoveVirtualEnvironment(string envName) + [Description("Remove the named Jupyter kernel. Missing environments or kernels produce a warning instead of an error.")] + [Input("name", "The name of the Jupyer kernel to remove.")] + [Input("run", "Run this method.")] + public static void RemoveJupyterKernel(string name, bool run = false) { - string envPath = Path.Combine(Query.DirectoryEnvironments(), envName); - if (Directory.Exists(envPath)) - Directory.Delete(envPath, true); + if (!Query.VirtualEnvironmentExists(name)) + { + Base.Compute.RecordWarning($"No Python environment named {name} exists."); + return; + } - RemoveKernel(envName); - } + PythonEnvironment venv = Query.VirtualEnvironment(name); + bool? exists = JupyterKernelExists(venv, name); + if (exists != true) + { + if (exists == false) + Base.Compute.RecordWarning($"No Jupyter kernel named {name} exists."); + return; + } - [Description("Remove all BHoM Python virtual environments.")] - public static void RemoveAllVirtualEnvironments() - { - DirectoryInfo di = new DirectoryInfo(Query.DirectoryEnvironments()); - foreach (DirectoryInfo dir in di.EnumerateDirectories()) + string cmd = $"\"{venv.Executable}\" -m jupyter kernelspec uninstall \"{name.ToLowerInvariant()}\" -f"; + + if (run) { - if (dir.Name != Query.ToolkitName()) - { - RemoveVirtualEnvironment(dir.Name); - } - + RunCommandStdout(command: cmd, hideWindows: true); + } + else + { + Base.Compute.RecordNote($"The following command will be executed: {cmd}"); } } - [Description("Remove the base install for the python version specified.")] - [Input("version", "The base python version to remove.")] - public static void RemoveBaseVersion(PythonVersion version = PythonVersion.v3_10) + [Description("Remove UV from the BHoM tools directory. Missing installations produce a warning; installations elsewhere are not removed.")] + [Input("run", "Run this method.")] + public static void RemoveUV(bool run = false) { - string basePath = Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName(), version.ToString()); - string baseInstaller = Path.Combine(basePath, "installer.exe"); + string toolsDirectory = Query.DirectoryTools(); - // If the installer does not exist and the folder does - assume a bad/invalid install and just delete the entire folder. - if (!File.Exists(baseInstaller) && Directory.Exists(basePath)) + if (!File.Exists(Query.UVExecutable())) { - Directory.Delete(basePath, true); + Base.Compute.RecordWarning($"UV is not currently installed in {toolsDirectory}."); return; } - else if (!Directory.Exists(basePath)) - // If the base path doesn't exist there is nothing to remove. - return; - using (Process uninstall = new Process() + if (run) { - StartInfo = new ProcessStartInfo() + try { - FileName = baseInstaller, - Arguments = "/uninstall /passive", - RedirectStandardError = true, - UseShellExecute = false, + Directory.Delete(toolsDirectory, true); } - }) - { - uninstall.StartInfo.Environment["PYTHONHOME"] = ""; - uninstall.Start(); - string stderr = uninstall.StandardError.ReadToEnd(); - uninstall.WaitForExit(); - if (uninstall.ExitCode != 0) + catch (IOException ex) { - BH.Engine.Base.Compute.RecordError($"Error uninstalling python: {stderr}"); - return; + Base.Compute.RecordError($"Failed to remove UV tools directory {toolsDirectory}: {ex.Message}"); } } - - // Finally remove base folder as the installer may have missed something. - Directory.Delete(basePath, true); + else + { + Base.Compute.RecordNote($"UV (and all other associated tools) will be removed from {toolsDirectory}."); + } } - [Description("Completely remove all BHoM-related Python environments and kernels.")] - public static void RemoveEverything() + [PreviousVersion("9.3", "BH.Engine.Python.Compute.RemoveEverything()")] + [Description("Completely remove all BHoM-related Python environments, kernels and tools.")] + public static void RemoveEverything(bool run = false) { - RemoveAllVirtualEnvironments(); - - //remove all python versions installed in base directory. - foreach (PythonVersion e in Enum.GetValues(typeof(PythonVersion))) + if ( run ) { - if (e != PythonVersion.Undefined) - RemoveBaseVersion(e); + foreach (string env in Directory.GetDirectories(Query.DirectoryEnvironments())) + { + RemoveVirtualEnvironment(Path.GetDirectoryName(env), run: run); + } + RemoveUV(run: run); } } } } - - - diff --git a/Python_Engine/Compute/RequirementsTxt.cs b/Python_Engine/Compute/RequirementsTxt.cs deleted file mode 100644 index 7084732d..00000000 --- a/Python_Engine/Compute/RequirementsTxt.cs +++ /dev/null @@ -1,91 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using BH.oM.Base.Attributes; -using System; -using System.ComponentModel; -using System.Diagnostics; -using System.IO; - -namespace BH.Engine.Python -{ - public static partial class Compute - { - [Description("Create a requirements.txt file from an existing Python executable.")] - [Input("executable", "The path to the Python executable to create the requirements.txt file from.")] - [Input("targetPath", "The path to the requirements.txt file to create. If not specified, the file will be created in the same directory as the executable.")] - [Output("requirementsTxt", "The path to the requirements.txt file.")] - public static string RequirementsTxt( - string executable, - string targetPath = null - ) - { - if (!File.Exists(executable)) - { - BH.Engine.Base.Compute.RecordError($"The executable '{executable}' does not exist."); - return null; - } - - if (string.IsNullOrEmpty(targetPath)) - { - targetPath = Path.Combine(Path.GetDirectoryName(executable), "requirements.txt"); - } - else - { - if (!Directory.Exists(Path.GetDirectoryName(targetPath))) - { - BH.Engine.Base.Compute.RecordError($"The executable '{executable}' does not exist."); - return null; - } - } - - System.Diagnostics.Process process = new System.Diagnostics.Process() - { - StartInfo = new System.Diagnostics.ProcessStartInfo() - { - FileName = executable, - Arguments = $"-m pip freeze", - UseShellExecute = false, - RedirectStandardError = true, - RedirectStandardOutput = true - } - }; - - process.StartInfo.Environment["PYTHONHOME"] = ""; - using (Process p = Process.Start(process.StartInfo)) - { - StreamWriter sr = new StreamWriter(targetPath); - sr.Write(p.StandardOutput.ReadToEnd()); - string standardError = p.StandardError.ReadToEnd(); - p.WaitForExit(); - if (p.ExitCode != 0) - BH.Engine.Base.Compute.RecordError($"Error creating requirements.txt from \"{executable}\".\n{standardError}"); - sr.Close(); - } - - return targetPath; - } - } -} - - - diff --git a/Python_Engine/Compute/Run.cs b/Python_Engine/Compute/Run.cs index 7340e693..633ca354 100644 --- a/Python_Engine/Compute/Run.cs +++ b/Python_Engine/Compute/Run.cs @@ -20,10 +20,9 @@ * along with this code. If not, see . */ -using BH.oM.Base; using BH.oM.Base.Attributes; -using BH.oM.Python; -using BH.oM.Python.Enums; +using BH.oM.Base; +using System; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; @@ -35,51 +34,114 @@ namespace BH.Engine.Python { public static partial class Compute { - [Description("Run a command via CMD and return stdout.")] + [Description("Run a command via CMD and return both output streams and the exit code.")] [Input("command", "The command to be run.")] [Input("hideWindows", "Set to True to hide cmd windows.")] [Input("startDirectory", "The directory in which the command should be run.")] [Input("timeoutMinutes", "A number of minutes beyond which this command will timeout.")] - [Output("stdout", "The StandardOutput from the command that was run. If the process failed, then StandardError will be returned here instead.")] - public static string RunCommandStdout(string command, bool hideWindows = true, string startDirectory = null, double timeoutMinutes = 5) + [MultiOutput(0, "stdout", "The command's standard output, including on failure.")] + [MultiOutput(1, "stderr", "The command's standard error, including on success.")] + [MultiOutput(2, "exitCode", "The process exit code. Zero indicates success; -1 indicates a timeout.")] + public static Output RunCommandStdout( + string command, + bool hideWindows = true, + string startDirectory = null, + double timeoutMinutes = 5 + ) { - System.Diagnostics.Process process = new System.Diagnostics.Process(); - - string commandMode = "/K"; - if (hideWindows) + if (string.IsNullOrWhiteSpace(startDirectory)) { - process.StartInfo.CreateNoWindow = true; - process.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden; - commandMode = "/C"; + startDirectory = Environment.CurrentDirectory; } - process.StartInfo.Environment["PYTHONHOME"] = ""; - process.StartInfo.FileName = "cmd.exe"; - process.StartInfo.RedirectStandardOutput = true; - process.StartInfo.RedirectStandardError = true; - process.StartInfo.UseShellExecute = false; - process.StartInfo.Arguments = $"{commandMode} {command}"; - process.Start(); - - // To avoid deadlocks, always read the output stream first and then wait. - string stdout = process.StandardOutput.ReadToEnd(); - string stderr = process.StandardError.ReadToEnd(); - - int millisecondsToWait = (int)timeoutMinutes * 60 * 1000; - process.WaitForExit(millisecondsToWait); - - if (process.ExitCode != 0) + + if (!Directory.Exists(startDirectory)) { - return stderr.Trim(); + throw new DirectoryNotFoundException( + string.Format("Working directory does not exist: '{0}'", startDirectory)); } - return stdout.Trim(); + using (Process process = new Process()) + { + process.StartInfo.FileName = "cmd.exe"; + process.StartInfo.Arguments = string.Format("/C \"{0}\"", command); + process.StartInfo.WorkingDirectory = startDirectory; + + process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.CreateNoWindow = hideWindows; + process.StartInfo.WindowStyle = hideWindows ? ProcessWindowStyle.Hidden : ProcessWindowStyle.Normal; + + // Avoid environment-variable interference. + process.StartInfo.EnvironmentVariables["PYTHONHOME"] = string.Empty; + process.StartInfo.EnvironmentVariables["PYTHONPATH"] = string.Empty; + + process.Start(); + + StreamReader stdoutReader = process.StandardOutput; + StreamReader stderrReader = process.StandardError; + + // Start reads without inheriting the host's UI synchronization context. + Task outputTask = Task.Run(async () => + { + Task stdoutTask = stdoutReader.ReadToEndAsync(); + Task stderrTask = stderrReader.ReadToEndAsync(); + + return await Task.WhenAll(stdoutTask, stderrTask).ConfigureAwait(false); + }); + + int timeoutMs = checked((int)(timeoutMinutes * 60 * 1000)); + + bool timedOut = !process.WaitForExit(timeoutMs); + if (timedOut) + { + // Kill descendants too, so inherited pipe handles cannot keep the reads open. + using (Process terminator = Process.Start(new ProcessStartInfo + { + FileName = "taskkill.exe", + Arguments = $"/PID {process.Id} /T /F", + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true + })) + { + Task killStdout = terminator.StandardOutput.ReadToEndAsync(); + Task killStderr = terminator.StandardError.ReadToEndAsync(); + terminator.WaitForExit(); + Task.WaitAll(killStdout, killStderr); + if (terminator.ExitCode != 0 && !process.HasExited) + throw new InvalidOperationException($"Could not terminate timed-out command: {killStderr.GetAwaiter().GetResult()}"); + } + process.WaitForExit(); + BH.Engine.Base.Compute.RecordError(new TimeoutException(), $"Command exceeded timeout of {timeoutMinutes} minutes.\nCommand: {command}"); + } + + // Ensure async reads have completed. + string[] results = outputTask.GetAwaiter().GetResult(); + + string stdout = results[0].Trim(); + string stderr = results[1].Trim(); + + int exitCode = timedOut ? -1 : process.ExitCode; + + if (!timedOut && exitCode != 0) + { + string message = $"Failed to execute {command} (exit code {exitCode}).\n{stdout}\n{stderr}"; + BH.Engine.Base.Compute.RecordError( + exception: new InvalidOperationException(message), + message: message); + } + + return new Output { Item1 = stdout, Item2 = stderr, Item3 = exitCode }; + } } [Description("Run a string containing Python code and return the output. The string passed should end with a \"print\" statement, and other print statements within it should be captured within the script to ensure only a single output string is passed back to this method.")] - [Input("env", "The Python environment with which to run the Python script.")] + [Input("venv", "The Python environment with which to run the Python script.")] [Input("pythonString", "The string containing the Python script.")] [Output("result", "The stdout data from the executed Python script.")] - public static string RunPythonString(this oM.Python.PythonEnvironment env, string pythonString) + public static string RunPythonString(this oM.Python.PythonEnvironment venv, string pythonString) { if (!pythonString.Split('\n').Last().Contains("print")) { @@ -93,12 +155,10 @@ public static string RunPythonString(this oM.Python.PythonEnvironment env, strin outputFile.WriteLine(pythonString); } - string cmd = $"{Modify.AddQuotesIfRequired(env.Executable)} {scriptFile}"; + string cmd = $"{Modify.AddQuotesIfRequired(venv.Executable)} {scriptFile}"; - return RunCommandStdout(cmd, hideWindows: true); + Output result = RunCommandStdout(command: cmd); + return result.Item3 == 0 ? result.Item1 : result.Item2; } } } - - - diff --git a/Python_Engine/Compute/UnzipFile.cs b/Python_Engine/Compute/UnzipFile.cs deleted file mode 100644 index 5c292a41..00000000 --- a/Python_Engine/Compute/UnzipFile.cs +++ /dev/null @@ -1,86 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using BH.oM.Base.Attributes; -using System; -using System.ComponentModel; -using System.IO; - -namespace BH.Engine.Python -{ - public static partial class Compute - { - //This method is no longer used by python toolkit, and perhaps should be removed or moved to the file toolkit instead. - [Description("Extract the contents of an archive.")] - [Input("archivePath", "The archive to extract.")] - [Input("targetDirectory", "The destination directory to extract into.")] - [Input("targetName", "An optional input to rename the archive directory.")] - [Input("delete", "An optional input to delete the archive after extraction.")] - public static void UnzipFile( - string archivePath, - string targetDirectory, - string targetName = null, - bool delete = false - ) - { - if (string.IsNullOrWhiteSpace(archivePath)) - { - BH.Engine.Base.Compute.RecordError($"The archivePath cannot be null or empty."); - } - - if (string.IsNullOrWhiteSpace(targetDirectory)) - { - BH.Engine.Base.Compute.RecordError($"The targetDirectory cannot be null or empty."); - } - - if (!File.Exists(archivePath)) - { - BH.Engine.Base.Compute.RecordError($"The archivePath '{archivePath}' does not exist."); - } - - if (!Directory.Exists(targetDirectory)) - { - BH.Engine.Base.Compute.RecordError($"The targetDirectory '{targetDirectory}' does not exist."); - } - - if (string.IsNullOrWhiteSpace(targetName)) - { - targetName = Path.GetFileNameWithoutExtension(archivePath); - } - - string targetPath = Path.Combine(targetDirectory, targetName); - - if (Directory.Exists(targetPath)) - { - BH.Engine.Base.Compute.RecordError($"The targetPath '{targetPath}' already exists."); - } - - System.IO.Compression.ZipFile.ExtractToDirectory(archivePath, targetPath); - - if (delete) - File.Delete(archivePath); - } - } -} - - - diff --git a/Python_Engine/Compute/UpdateBHoMPackages.cs b/Python_Engine/Compute/UpdateBHoMPackages.cs index c1f3024f..5fbebc2b 100644 --- a/Python_Engine/Compute/UpdateBHoMPackages.cs +++ b/Python_Engine/Compute/UpdateBHoMPackages.cs @@ -20,91 +20,23 @@ * along with this code. If not, see . */ -using BH.oM.Base; using BH.oM.Base.Attributes; using BH.oM.Python; -using BH.oM.Python.Enums; -using System; -using System.Collections.Generic; using System.ComponentModel; -using System.Diagnostics; using System.IO; -using System.Linq; -using System.Text; namespace BH.Engine.Python { public static partial class Compute { - [Description("Using the environment name (and version if the environment name is the base python environment), directly install/update missing BHoM packages.")] - [Input("environmentName", "The name of the environment to update.")] - [Input("version", "If the environment name is the name of the base python environment, the version that needs updating.")] - public static void UpdateBHoMPackages(this string environmentName, PythonVersion version = PythonVersion.Undefined) + [Description("Perform an update on the provided Python environment.")] + [Input("venv", "The Python environment to update.")] + public static void UpdateBHoMPackages(this PythonEnvironment venv) { - //construct the expected PythonEnvironment and pass to UpdateBHoMPackages - PythonEnvironment env = new PythonEnvironment() - { - Name = environmentName, - }; + string cmd = $"\"{Query.UVExecutable()}\" sync"; + string directory = Path.Combine(Query.DirectoryEnvironments(), venv.Name); - //python version is only used if the environment name is this toolkits name. - if (environmentName == Query.ToolkitName()) - { - if (version == PythonVersion.Undefined) - { - BH.Engine.Base.Compute.RecordError("The version must be specified when updating BHoM packages for a base python environment. No updates have occurred."); - return; - } - - env.Executable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe"); - } - else - env.Executable = Query.VirtualEnvironmentExecutable(environmentName); - - UpdateBHoMPackages(env); - } - - /***************************************************/ - - [Description("Using a PythonEnvironment, directly install/update missing BHoM packages.")] - [Input("environment", "The python environment to update.")] - public static void UpdateBHoMPackages(this PythonEnvironment environment) - { - if (!File.Exists(environment.Executable)) - { - BH.Engine.Base.Compute.RecordError($"Given environment or base install {environment.Executable} does not exist."); - return; - } - - //`python -m pip list -e --format json` lists all environment packages that are editable installs. - //As all BHoM packages are editable installs, this handily lists all the packages that should be updated if necessary. - //side effect of updating other editable installs if they have changed, but I suspect this won't be a problem as anyone who has these would update packages manually instead. - System.Diagnostics.Process process = new System.Diagnostics.Process() - { - StartInfo = new System.Diagnostics.ProcessStartInfo() - { - FileName = environment.Executable, - Arguments = $"-m pip list -e --format json", - UseShellExecute = false, - RedirectStandardOutput = true, - } - }; - - process.StartInfo.Environment["PYTHONHOME"] = ""; - string stdOut; - - using (Process p = Process.Start(process.StartInfo)) - { - stdOut = p.StandardOutput.ReadToEnd(); - p.WaitForExit(); - } - - stdOut = stdOut.TrimEnd('\r', '\n'); - - IEnumerable objs = Serialiser.Convert.FromJsonArray(stdOut).OfType(); - - foreach (CustomObject obj in objs) - InstallPackageLocal(environment, (string)obj.CustomData["editable_project_location"]); + RunCommandStdout(cmd, startDirectory: directory); } } -} \ No newline at end of file +} diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index 6ed9dde7..18d592c1 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -1,4 +1,4 @@ -/* +/* * This file is part of the Buildings and Habitats object Model (BHoM) * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. * @@ -20,120 +20,139 @@ * along with this code. If not, see . */ -using BH.Engine.Base; using BH.oM.Base.Attributes; using BH.oM.Python; -using BH.oM.Python.Enums; -using System.Collections.Generic; +using System; using System.ComponentModel; -using System.Diagnostics; using System.IO; -using System.Linq; +using System.Text.RegularExpressions; namespace BH.Engine.Python { public static partial class Compute { - [Description("Install a Python Virtual Environment of a given version and name.")] + [PreviousVersion("9.3", "BH.Engine.Python.Compute.VirtualEnvironment(PythonVersion, string, bool)")] + [Description("Create a Python Virtual Environment of a given version and name.")] [Input("version", "The version of Python to use for this environment.")] - [Input("name", "The name of this virtualenv.")] + [Input("name", "The name of this venv.")] [Input("reload", "Reload the virtual environment rather than recreating it, if it already exists.")] - [Output("env", "The exectuble for the resultant virtualenv.")] - public static PythonEnvironment VirtualEnvironment(this PythonVersion version, string name, bool reload = true) + [Output("venv", "The resultant venv.")] + public static PythonEnvironment VirtualEnvironment(string version, string name, bool reload = true) { - if (!Query.IsValidEnvironmentName(name)) + if (Query.VirtualEnvironmentExists(name)) { - BH.Engine.Base.Compute.RecordError("A BHoM Python virtual environment cannot cannot contain invalid filepath characters."); - return null; + if (reload) + { + PythonEnvironment existingVenv = new PythonEnvironment() { Executable = Query.VirtualEnvironmentExecutable(name), Name = name }; + string existingVersion = Query.Version(existingVenv); + if (existingVersion == null || (existingVersion != version && !existingVersion.StartsWith(version + ".", StringComparison.Ordinal))) + { + BH.Engine.Base.Compute.RecordError($"The requested environment already exists using Python {existingVersion}, but is being requested with Python {version}. Either provide a different name, a matching version, or remove the existing environment."); + return null; + } + return existingVenv; + } + else + { + RemoveVirtualEnvironment(name: name, run: true); + } } - if (version == PythonVersion.Undefined) - { - BH.Engine.Base.Compute.RecordError("Please provide a valid Python version."); + InstallUV(true); // ensure UV is installed + + string directory = Path.Combine(Query.DirectoryEnvironments(), name); + Directory.CreateDirectory(directory); + + string cmd = $"\"{Query.UVExecutable()}\" venv \"{Path.Combine(directory, ".venv")}\" --python \"{version}\""; + if (RunCommandStdout(command: cmd, startDirectory: directory).Item3 != 0) return null; - } - string targetExecutable = Query.VirtualEnvironmentExecutable(name); - string targetDirectory = Query.VirtualEnvironmentDirectory(name); - string versionExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe"); + PythonEnvironment venv = new PythonEnvironment() { Executable = Query.VirtualEnvironmentExecutable(name), Name = name }; - bool exists = Query.VirtualEnvironmentExists(name); + InstallJupyterKernel(venv: venv); - if (exists && reload) + return venv; + } + + [Description("Create a Python Virtual Environment from a TOML file in the named BHoM environment directory. Before installation, set the project's version to the current BHoM version.")] + [Input("tomlFile", "The path to a TOML file containing venv configuration.")] + [Input("reload", "Reload the virtual environment rather than recreating it, if it already exists.")] + [Output("venv", "The resultant venv.")] + public static PythonEnvironment VirtualEnvironment(string tomlFile, bool reload = true) + { + string name = Directory.GetParent(tomlFile).Name; + string directory = Path.Combine(Query.DirectoryEnvironments(), name); + + // a package must have an associated TOML file, a README.md and a src/ + if (!File.Exists(tomlFile)) { - PythonEnvironment env = new PythonEnvironment() { Name = name, Executable = targetExecutable }; - UpdateBHoMPackages(env); - return env; + Base.Compute.RecordError($"TOML file not found at {tomlFile}"); + return null; } - if (exists && !reload) + string readmeFile = Path.Combine(directory, "README.md"); + if (!File.Exists(readmeFile)) { - // remove all existing environments and kernels - RemoveVirtualEnvironment(name); - RemoveKernel(name); + Base.Compute.RecordError($"README.md file not found at {readmeFile}"); + return null; } - if (!File.Exists(versionExecutable)) + string toml = File.ReadAllText(tomlFile); + Match project = Regex.Match(toml, @"(?ms)^[ \t]*\[project\][ \t]*(?:#[^\r\n]*)?\r?\n(?.*?)(?=^[ \t]*\[|\z)"); + string packageName = Regex.Match(project.Groups["content"].Value, + @"(?m)^[ \t]*name[ \t]*=[ \t]*['""](?[^'""\r\n]+)['""]").Groups["name"].Value; + if (!project.Success || string.IsNullOrWhiteSpace(packageName)) { - // The output here should be the same, but to be sure replace the value. - BH.Engine.Base.Compute.RecordNote($"The base environment for the requested version {version} has been installed as it was not present."); - versionExecutable = BasePythonEnvironment(version, run: true)?.Executable; - - if (versionExecutable == null) // If the executable is null, then python wasn't installed correctly, so return null - the error message for downloading the version should suffice. - return null; + Base.Compute.RecordError($"A [project] section with a package name is required in {tomlFile}."); + return null; } - // create the venv from the base environment - Process process = new Process() + string packageFile = Path.Combine(directory, "src", packageName, "__init__.py"); + if (!File.Exists(packageFile)) { - StartInfo = new ProcessStartInfo() - { - FileName = Modify.AddQuotesIfRequired(versionExecutable), - Arguments = $"-m virtualenv --python={Modify.AddQuotesIfRequired(versionExecutable)} {Modify.AddQuotesIfRequired(targetDirectory)}", - RedirectStandardError = true, - UseShellExecute = false, - } - }; - - process.StartInfo.Environment["PYTHONHOME"] = ""; + Base.Compute.RecordError($"A Python package must contain a .\\src\\{packageName}\\__init__.py file."); + return null; + } - using (Process p = Process.Start(process.StartInfo)) + string projectContent = project.Groups["content"].Value; + Regex versionField = new Regex(@"(?m)^(?[ \t]*version[ \t]*=[ \t]*)(?['""])[^'""\r\n]*\k"); + if (!versionField.IsMatch(projectContent)) { - string standardError = p.StandardError.ReadToEnd(); - p.WaitForExit(); - if (p.ExitCode != 0) - BH.Engine.Base.Compute.RecordError($"Error creating virtual environment.\n{standardError}"); + Base.Compute.RecordError($"A static [project] version is required in {tomlFile}."); + return null; } - // install ipykernel, pytest and black into new virtualenv - InstallPackages(targetExecutable, new List() { "ipykernel", "pytest", "black" }); - - // register the virtualenv with the base environment - Process process2 = new Process() + if (Query.VirtualEnvironmentExists(name)) { - StartInfo = new ProcessStartInfo() + if (reload) + { + PythonEnvironment existingVenv = new PythonEnvironment() { Executable = Query.VirtualEnvironmentExecutable(name), Name = name }; + return existingVenv; + } + else { - FileName = targetExecutable, - Arguments = $"-m ipykernel install --name={name} --user", //--user adds the kernel to %appdata%/jupyter/kernels, rather than to the virtualenv/share/jupyter/kernels - RedirectStandardError = true, - UseShellExecute = false, + RemoveJupyterKernel(name, run: true); + // Retain the TOML file and package sources while recreating the environment. + Directory.Delete(Path.Combine(directory, ".venv"), true); } - }; + } - process2.StartInfo.Environment["PYTHONHOME"] = ""; + string updatedProject = versionField.Replace(projectContent, + match => match.Groups["prefix"].Value + $"\"{BH.Engine.Base.Query.BHoMVersion()}\"", 1); + Group content = project.Groups["content"]; + File.WriteAllText(tomlFile, toml.Substring(0, content.Index) + updatedProject + toml.Substring(content.Index + content.Length)); - using (Process p = Process.Start(process2.StartInfo)) - { - string standardError = p.StandardError.ReadToEnd(); - p.WaitForExit(); - if (p.ExitCode != 0) - BH.Engine.Base.Compute.RecordError($"Error registering the \"{name}\" virtual environment.\n{standardError}"); - } + InstallUV(true); // ensure UV is installed - return new PythonEnvironment() { Executable = targetExecutable, Name = name }; - } - } -} + string cmd = $"\"{Query.UVExecutable()}\" sync --all-packages"; + if (RunCommandStdout(cmd, startDirectory: directory).Item3 != 0) + return null; + PythonEnvironment venv = new PythonEnvironment() { Executable = Query.VirtualEnvironmentExecutable(name), Name = name }; + InstallJupyterKernel(venv: venv); + return venv; + } + } +} diff --git a/Python_Engine/Convert/ToPython.cs b/Python_Engine/Convert/ToPython.cs deleted file mode 100644 index 193669df..00000000 --- a/Python_Engine/Convert/ToPython.cs +++ /dev/null @@ -1,42 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BH.oM.Base.Attributes; - -namespace BH.Engine.Python -{ - public static partial class Convert - { - // TODO - REMOVE THIS METHOD, NO LONGER REQUIRED, PLUS IT HAS AN OLD ToBeRemoved TAG - [ToBeRemoved("5.3", "This method included in order to fool versioning into playing nicely.")] - public static bool ToPython(this T[,] input) - { - return false; - } - } -} - diff --git a/Python_Engine/Modify/AddQuotesIfRequired.cs b/Python_Engine/Modify/AddQuotesIfRequired.cs index ad54d616..fcd31693 100644 --- a/Python_Engine/Modify/AddQuotesIfRequired.cs +++ b/Python_Engine/Modify/AddQuotesIfRequired.cs @@ -44,8 +44,3 @@ public static string AddQuotesIfRequired(this string path) } } } - - - - - diff --git a/Python_Engine/Python/Dockerfile b/Python_Engine/Python/Dockerfile index dde75b24..e7a25a8b 100644 --- a/Python_Engine/Python/Dockerfile +++ b/Python_Engine/Python/Dockerfile @@ -1,22 +1,26 @@ -FROM python:3.10-slim -COPY . /Python_Toolkit +# syntax=docker/dockerfile:1 -#requires build argument --build-context bhom-assemblies C:/ProgramData/BHoM/Assemblies -COPY --from=bhom-assemblies . /bin/BHoM/Assemblies +FROM ghcr.io/astral-sh/uv:latest AS uv +FROM mcr.microsoft.com/dotnet/aspnet:10.0-bookworm-slim AS dotnet -#install .net core 8.0, and set the pythonnet runtime to the .net core CLR -RUN apt-get update \ - && apt-get install -y wget - -RUN wget https://packages.microsoft.com/config/debian/13/packages-microsoft-prod.deb -O packages-microsoft-prod.deb -RUN dpkg -i packages-microsoft-prod.deb -RUN rm packages-microsoft-prod.deb +FROM python:3.10-slim-bookworm -RUN apt-get update \ - && apt-get install -y aspnetcore-runtime-10.0 +COPY --from=uv /uv /opt/BHoM/Extensions/Python/tools/uv +COPY --from=dotnet /usr/share/dotnet /usr/share/dotnet -ENV PYTHONNET_RUNTIME="coreclr" +ENV DOTNET_ROOT=/usr/share/dotnet \ + PYTHONNET_RUNTIME=coreclr \ + UV_PYTHON_DOWNLOADS=never \ + UV_LINK_MODE=copy \ + PATH="/opt/BHoM/Extensions/Python/Python_Toolkit/.venv/bin:/opt/BHoM/Extensions/Python/tools:/usr/share/dotnet:${PATH}" \ + PYTHONPATH="/opt/BHoM/Assemblies" -RUN pip3 install ./Python_Toolkit +WORKDIR /opt/BHoM/Extensions/Python/Python_Toolkit -RUN rm ./Python_Toolkit -rf \ No newline at end of file +# Requires the bhom-assemblies named build context. +COPY --from=bhom-assemblies . /opt/BHoM/Assemblies + +COPY pyproject.toml README.md ./ +COPY src ./src + +RUN /opt/BHoM/Extensions/Python/tools/uv sync --no-dev --no-editable diff --git a/Python_Engine/Python/README.md b/Python_Engine/Python/README.md index bf482e26..e69de29b 100644 --- a/Python_Engine/Python/README.md +++ b/Python_Engine/Python/README.md @@ -1,262 +0,0 @@ -# Python_Toolkit - -The code contained here is intended to be used in conjunction with the Python_Toolkit, as part of a wider [BHoM](https://bhom.xyz/) installation. Its main purpose is to augment capabilities of the BHoM. - -# BHoM Python environments - -_Where guidelines are not explicitly given, code style standards and procedures should refer to the wider [BHoM code compliance guidance](https://github.com/BHoM/documentation/wiki/Code-Compliance)._ - -## Creation of a Python environment - -A BHoM toolkit can be associated with a Python environment. The toolkit needs to have a location where Python code associated with that toolkit exists - so the first step is in creating that location and structure. - -**In the example below we'll go through creating a Python environment for the `Example_Toolkit`**. - -### Structure - -All Python code within BHoM should be placed within the hosting toolkit, inside its `*_Engine` folder. Within this folder, the following directory structure should be created (files have been included here within `./example_toolkit`, though these are purely indicative and struictures can vary based on toolkit need): - -``` -. -├── ... -├── Example_Engine -│ └── Python -| ├── README.md -| ├── requirements.txt -| ├── setup.py -| ├── src -| | └── example_toolkit -| | ├── __init__.py -| | ├── module_a -| | | ├── __init__.py -| | | └── ... -| | └── ... -| └── tests -| ├── __init__.py -| ├── test_module_a.py -| └── ... -└── ... -``` - -### `setup.py` - -The `setup.py` file must be created alongside the code within a toolkit. This should contain teh following: - -```python -## setup.py ## -from pathlib import Path - -import setuptools -from win32api import HIWORD, LOWORD, GetFileVersionInfo - -TOOLKIT_NAME = "Example_Toolkit" - -def _bhom_version() -> str: - """Return the version of BHoM installed (using the BHoM.dll in the root BHoM directory.""" - info = GetFileVersionInfo("C:/ProgramData/BHoM/Assemblies/BHoM.dll", "\\") # pylint: disable=[no-name-in-module] - ms = info["FileVersionMS"] - ls = info["FileVersionLS"] - return f"{HIWORD(ms)}.{LOWORD(ms)}.{HIWORD(ls)}.{LOWORD(ls)}" # pylint: disable=[no-name-in-module] - -BHOM_VERSION = _bhom_version() - -here = Path(__file__).parent.resolve() -long_description = (here / "README.md").read_text(encoding="utf-8") -requirements = [ - i.strip() - for i in (here / "requirements.txt").read_text(encoding="utf-8-sig").splitlines() -] - -setuptools.setup( - name=TOOLKIT_NAME.lower(), - author="BHoM", - author_email="bhombot@burohappold.com", - description=f"A Python library that contains code intended to be used by the {TOOLKIT_NAME} Python environment for BHoM workflows.", - long_description=long_description, - long_description_content_type="text/markdown", - url=f"https://github.com/BHoM/{TOOLKIT_NAME}", - package_dir={"": "src"}, - packages=setuptools.find_packages(where="src", exclude=['tests']), - install_requires=requirements, - version=BHOM_VERSION, -) -``` - -### Enabling installation - -Two methods are required in the toolkit to which Python code is being added. The first is a `Query` method for the name of that toolkit: - -```C# -using BH.oM.Base.Attributes; - -using System.ComponentModel; - -namespace BH.Engine.Example -{ - public static partial class Query - { - [Description("Get the name of the current toolkit.")] - [Output("name", "The name of the current toolkit.")] - public static string ToolkitName() - { - return "Example_Toolkit"; - } - } -} -``` - -And the second is a `Compute` method called `ExampleToolkitPythonEnvironment` which installs the toolkits associated Python environment. In the method below, Python v3.7.9 is being used for this toolkits Python environment: - -```C# -using BH.oM.Base.Attributes; -using BH.oM.Python; - -using System.ComponentModel; - -namespace BH.Engine.Example -{ - public static partial class Compute - { - [Description("Install the Python virtualenv for the Example_Toolkit.")] - [Input("run", "Run the installation process for this BHoM Python environment.")] - [Output("env", "The Example_Toolkit Python environment, including any locally referenced BHoM code.")] - public static PythonEnvironment ExampleToolkitPythonEnvironment(bool run = false) - { - if (run) - { - return BH.Engine.Python.Compute.InstallVirtualenv( - name: Query.ToolkitName(), - pythonVersion: BH.oM.Python.Enums.PythonVersion.v3_7_9, // the version of Python to be used for this toolkits Python environment - localPackage: Path.Combine(Engine.Python.Query.CodeDirectory(), Query.ToolkitName()), - run: run - ); - } - return null; - } - } -} -``` - -Variations upon this can be made to reference external environments with known Python versions and installed packages also - with the [LadybugTools_Toolkit](https://github.com/BHoM/LadybugTools_Toolkit/blob/c005dc901459c4b779b329b65406d4a6c88f0965/LadybugTools_Engine/Compute/LadybugToolsToolkitPythonEnvironment.cs) showing how this is done. - -## Structure of Python code - -Unless otherwise specified, code style must follow guidance from [PEP8](https://peps.python.org/pep-0008/), unless otherwise stated in this document or in [BHoM C# guidance](https://github.com/BHoM/documentation/wiki/Coding-Style) (though some aspects of C# guidance are not applicable here due to differences between the two languages). - -### BHoM/Python specific style guidance - -- One method or class per file. -- As much as possible, your code should be logically organised into a collection of directories and subdirectories. -- Helper methods often don't fit into specific workflows and exist outside of packaged Python modules. These should be placed in a `./helpers` directory in the root of the toolkits Python code. -- Files named in snake_case. For example, a class called `AThing`, would be in a file called `a_thing.py`, and a method called `do_stuff` would be in a files called `do_stuff.py`. -- Imports at the top of files must be absolute, to remove all ambiguity. -- Type hints are required on all functions and classes (both inputs and outputs). -- Docstrings must be added for all classes and methods. If the method is wrapped by a class, then only a description of that method is required as a minimum (unless the method is a dunder method). -- Docstrings should be provided in [Google format](https://google.github.io/styleguide/pyguide.html#383-functions-and-methods). -- Default values should only be used where the defaults given are expected by a typical user of that code. If in doubt do not include a default. -- Where defaults are given, these are best placed in the method/class instantiation rather than reassigning to that variable upon checking if variable "is None" later in the code. -- Errors should be handled gracefully. When errors happen, the user should be told what went wrong and how to fix it. Broad Exceptions are allowed, although specific errors are preferable. Errors should also be captured at the lowest-possible level. - -## Testing - -The following guidelines are given for implementation of Python code testing. It is anticipated that the majority of testing can be undertaken automatically without need for human interaction. This does increase the time required to develop code - but will result in far better code overall as a result. Everything that can be reasonably tested should be done so. Dispensation may be given from the CI/CD lead where appropriate following suitable justification made by the code developer. - -The testing framework used in all BHoM Python code is `pytest`. In each BHoM toolkit, in the base of that toolkit is a ./.ci directory wherein tests should be placed. For example, in the `Example_Toolkit`, tests would be located in `./Example_Toolkit/.ci/unit_tests/python`. Tests should be created by the person that writes the code that is being tested, and subject to review by those undertaking the PR review. A passing unit test is required for PR approval. The reviewer needs to ensure that the testing procedure is sufficiently comprehensive as part of their approval. - -An example is given below for an appropriate candidate for testing: - -```python -## example_code.py ## -from typing import Union -def add(a: Union[int, float], b: Union[int, float]) -> float: - """Add two numbers together. - - Args: - a (Union[int, float]): The first number. - b (Union[int, float]): The second number. - - Returns: - Union[int, float]: The result of adding the inputs together. - """ - if (not isinstance(a, (int, float))) and (not isinstance(b, (int, float)): - raise TypeError(f"The combination of dtypes ({type(a)}, {type(b)}) passed are not valid for this method.") - return a + b -``` - -... with testing procedures defined to check that this method works as expected ... - -```python -## test_example_code.py ## -import pytest -from example_code import add - -def test_add(): - a = 1.5 - b = 2.5 - assert add(a, b) == 4.0 - -def test_add_fails(): - with pytest.raises(TypeError) as e_info: - add("1", 1) - -``` - -... and the command used to test this would be ... - -``` -python -m pytest ./test_example_code.py -``` - -Tests should be written so that: - -- Cases where a method runs correctly and returns a pre-defined result pass. -- Cases where the method handles errors gracefully and raises an expected error pass. -- In complex processes, tests may not be possible for all methods; however, it is expected that the lowest methods in that process are tested to reduce risk of error cascading through these processes. -- In complex processes that rely on external programs, assertions should be made at the beginning of testing that those programs are available to the testing methods. -- Where stochastic results are expected, use "fuzzy matching" to check for closeness of results (see [pytest.approx](https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest-approx)) - -# Recommended development environment - -Python in the context of BHoM is intended to be used where C# may not be as rapid or best placed to give results. As such, it can be considered more of a "scripting" language - with scope for formalisation into properly formatted and discretised code as part of BHoM toolkits. Ultimately it is up to the developer how to write their code, but the following setup is suggested for those just starting. - -## IDE - -Use VSCode for Python code development. This editor provides extensibility via packages which enable the automation of a lot of repetitive actions. The plugins that are recommended are: - -- python -- pylance -- Prettier -- autoDocstring - -Additionally, the following user settings can be added to enable auto-formatting of code upon saving to PEP8 compliance using "black" (a requirement of all BHoM Python code). - -```json -{ - "python.formatting.provider": "black", - "python.defaultInterpreterPath": "C:/ProgramData/BHoM/Extensions/PythonEnvironments/Example_Toolkit/python.exe", - "editor.formatOnType": true, - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.fixAll": true, - "source.organizeImports": true, - "source.sortMembers": true - }, - "python.formatting.blackPath": "C:/ProgramData/BHoM/Extensions/PythonEnvironments/Python_Toolkit/Scripts/black.exe", - "python.linting.pylintArgs": ["--disable=C0114"] -} -``` - -## Scripting/notebooks - -Jupyter notebooks provide a good testing environment for Python code development. The registered ipykernel associated with each BHoM Python environment enables creation of environment specific notebooks with access to all code contained therein. Once you've got the base BHoM Python environment installed, you can run a notebook server using - -```bash -C:/ProgramData/BHoM/Extensions/PythonEnvironments/Python_Toolkit/python.exe -m jupyter lab -``` - -During development it can be useful to reload changes in code rather than restarting the kernel each time you modify the source code. To do this, include the following in the first cell in a notebook and changes you make in the referenced \*.py files will be brought through into the notebook environment. - -```python -%load_ext autoreload -%autoreload 2 -``` diff --git a/Python_Engine/Python/pyproject.toml b/Python_Engine/Python/pyproject.toml index 6db0bd6c..b388b131 100644 --- a/Python_Engine/Python/pyproject.toml +++ b/Python_Engine/Python/pyproject.toml @@ -1,54 +1,48 @@ [project] name = "python_toolkit" +version = "999.999.999" +description = "The core BHoM Python toolkit environment, containing essential tools and utilities." readme = "README.md" -dynamic = ["version", "description"] +requires-python = ">=3.10,<3.11" authors = [ { name = "BHoM", email = "bhombot@burohappold.com" } ] dependencies = [ - "case-converter", - "jupyterlab", "matplotlib", - "numpy", + "numpy", "pandas", "plotly", - "pytest>=8.3.5", - "pytest-cov>=6.0.0", - "pytest-order", - "virtualenv", "pythonnet", + "pytz", + "pywin32; sys_platform == 'win32'", ] -[urls] -source = "https://github.com/BHoM/Python_Toolkit" - [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" + [dependency-groups] dev = [ - "black", - "ipykernel>=6.29.5", - "pylint>=3.3.5", - "pytest>=8.3.5", - "pytest-cov>=6.0.0", - "pytest-order", - "sphinx>=8.1.3", - "sphinx-bootstrap-theme>=0.8.1", - "sphinxcontrib-fulltoc>=1.2.0", - "sphinxcontrib-napoleon>=0.7", + "flask", + "furo", + "interrogate", + "ipykernel", + "line-profiler", + "memory-profiler", + "mypy", + "pydata-sphinx-theme", + "pydocstyle", + "pylint", + "pytest-benchmark", + "pytest-cov", + "pytest-dependency", + "pytest-xdist", + "pytest", + "ruff", + "sphinx-bootstrap-theme", + "sphinx", + "sphinx-autoapi", + "sphinxcontrib-fulltoc", + "sphinxcontrib-napoleon" ] - -[tool.setuptools] -package-dir = {"" = "src"} - -[tool.setuptools.packages.find] -where = ["src"] -exclude = ["tests"] - -[tool.setuptools.package-data] -"*" = ["*.mplstyle"] - -[tool.pytest.ini_options] -pythonpath = "src" diff --git a/Python_Engine/Python/run_tests.bat b/Python_Engine/Python/run_tests.bat deleted file mode 100644 index 18e46305..00000000 --- a/Python_Engine/Python/run_tests.bat +++ /dev/null @@ -1,7 +0,0 @@ -SET PYTHON_EXE=C:\ProgramData\BHoM\Extensions\PythonEnvironments\Python_Toolkit\v3_10\python.exe -SET TEST_DIR=C:\ProgramData\BHoM\Extensions\PythonCode\Python_Toolkit\tests - -@REM C:\ProgramData\BHoM\Extensions\PythonEnvironments\Python_Toolkit\v3_10\python.exe -m pytest --cov-report term --cov-report html:cov_html --cov python_toolkit -v C:\ProgramData\BHoM\Extensions\PythonCode\Python_Toolkit\tests - -"%PYTHON_EXE%" -m pytest --cov-report term --cov-report html:cov_html --cov python_toolkit -v "%TEST_DIR%" -cmd /k \ No newline at end of file diff --git a/Python_Engine/Python/scripts/build_docs.ps1 b/Python_Engine/Python/scripts/build_docs.ps1 new file mode 100644 index 00000000..6fed7fc4 --- /dev/null +++ b/Python_Engine/Python/scripts/build_docs.ps1 @@ -0,0 +1,50 @@ +param ( + [switch]$Serve, + [ValidateRange(1, 65535)] + [int]$Port = 8000 +) + +. "$PSScriptRoot\shared.ps1" + +function Build-Documentation { + param ( + [switch]$Serve, + [int]$Port + ) + + $functionName = $MyInvocation.MyCommand.Name + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $docsDirectory = Join-Path $workspaceRoot "docs" + $configFile = Join-Path $docsDirectory "conf.py" + $outputDirectory = Join-Path $docsDirectory "_build\html" + $pythonExecutable = $null + + try { + if (-not (Test-Path $configFile -PathType Leaf)) { + throw "Sphinx configuration was not found at $configFile." + } + + $pythonExecutable = Get-LocalPythonExecutable + Write-InfoMessage -Message "Building HTML documentation using $configFile." -FunctionName $functionName + & $pythonExecutable -m sphinx -b html -c $docsDirectory $docsDirectory $outputDirectory + if ($LASTEXITCODE -ne 0) { + throw "Sphinx build failed with exit code $LASTEXITCODE." + } + Write-InfoMessage -Message "Documentation was built at $outputDirectory." -FunctionName $functionName + + if ($Serve) { + Write-InfoMessage -Message "Serving documentation at http://127.0.0.1:$Port/ (press Ctrl+C to stop)." -FunctionName $functionName + & $pythonExecutable -m http.server $Port --bind 127.0.0.1 --directory $outputDirectory + if ($LASTEXITCODE -ne 0) { + throw "The documentation server exited with code $LASTEXITCODE." + } + } + } + catch { + Write-ErrorMessage -Message "Documentation build or serve failed. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } +} + +if ($MyInvocation.InvocationName -ne '.') { + Build-Documentation -Serve:$Serve -Port $Port +} diff --git a/Python_Engine/Python/scripts/install.ps1 b/Python_Engine/Python/scripts/install.ps1 new file mode 100644 index 00000000..ab94076f --- /dev/null +++ b/Python_Engine/Python/scripts/install.ps1 @@ -0,0 +1,50 @@ +param ( + [switch]$Run +) + +. "$PSScriptRoot\shared.ps1" + +function Install-Package { + $functionName = $MyInvocation.MyCommand.Name + + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $locationPushed = $false + $toolkitName = "Python Toolkit" + $installationComplete = $false + + try { + Push-Location $workspaceRoot + $locationPushed = $true + $tomlFile = Get-TomlFile + $toolkitName = Get-ToolkitName + Write-InfoMessage -Message "Starting installation process for $toolkitName." -FunctionName $functionName + + Write-InfoMessage -Message "Preparing UV and its command-line tools." -FunctionName $functionName + Install-UV + + Write-InfoMessage -Message "Syncing the Python environment from $tomlFile." -FunctionName $functionName + Sync-PythonEnvironment + + Write-InfoMessage -Message "Registering the Jupyter kernel." -FunctionName $functionName + Register-Kernel -KernelName $toolkitName + $installationComplete = $true + } + catch { + Write-ErrorMessage -Message "Installation failed for $toolkitName. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } + finally { + if ($locationPushed) { + Pop-Location + } + if ($installationComplete) { + Write-InfoMessage -Message "Installation process for $toolkitName completed successfully." -FunctionName $functionName + } + else { + Write-WarningMessage -Message "Installation process for $toolkitName did not complete." -FunctionName $functionName + } + } +} + +if ($Run -or $MyInvocation.InvocationName -ne '.') { + Install-Package +} diff --git a/Python_Engine/Python/scripts/pre_commit.ps1 b/Python_Engine/Python/scripts/pre_commit.ps1 new file mode 100644 index 00000000..8b9198c0 --- /dev/null +++ b/Python_Engine/Python/scripts/pre_commit.ps1 @@ -0,0 +1,35 @@ + +param ( + [switch]$Run, + [switch]$IncludeSkipped +) +. "$PSScriptRoot\shared.ps1" + +function Format-PythonFiles { + param ( + [switch]$IncludeSkipped + ) + + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $pushedLocation = $false + + try { + Push-Location $workspaceRoot + $pushedLocation = $true + + Write-InfoMessage -Message "Formatting Python files in ./src" -FunctionName $MyInvocation.MyCommand.Name + & (Get-UVExecutable) run ruff format .\src + Write-InfoMessage -Message "Formatting Python files in ./tests" -FunctionName $MyInvocation.MyCommand.Name + & (Get-UVExecutable) run ruff format .\tests + + } + finally { + if ($pushedLocation) { + Pop-Location + } + } +} + +if ($Run -or $MyInvocation.InvocationName -ne '.') { + Format-PythonFiles +} diff --git a/Python_Engine/Python/scripts/run_tests.ps1 b/Python_Engine/Python/scripts/run_tests.ps1 new file mode 100644 index 00000000..9e1eccff --- /dev/null +++ b/Python_Engine/Python/scripts/run_tests.ps1 @@ -0,0 +1,49 @@ + +param ( + [switch]$Run, + [switch]$IncludeSkipped +) +. "$PSScriptRoot\shared.ps1" + +function Test-Code { + param ( + [switch]$IncludeSkipped + ) + + $functionName = $MyInvocation.MyCommand.Name + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $previousSkipLongPytest = $env:SKIP_LONG_PYTEST + $pushedLocation = $false + + try { + Push-Location $workspaceRoot + $pushedLocation = $true + + $pythonExecutable = Get-LocalPythonExecutable + Write-InfoMessage -Message "Using Python executable: $pythonExecutable" -FunctionName $functionName + + if ($IncludeSkipped) { + $env:SKIP_LONG_PYTEST = "1" + } + else { + Remove-Item Env:SKIP_LONG_PYTEST -ErrorAction SilentlyContinue + } + + & $pythonExecutable -m pytest + } + finally { + if ($null -eq $previousSkipLongPytest) { + Remove-Item Env:SKIP_LONG_PYTEST -ErrorAction SilentlyContinue + } + else { + $env:SKIP_LONG_PYTEST = $previousSkipLongPytest + } + if ($pushedLocation) { + Pop-Location + } + } +} + +if ($Run -or $MyInvocation.InvocationName -ne '.') { + Test-Code -IncludeSkipped:$IncludeSkipped +} diff --git a/Python_Engine/Python/scripts/shared.ps1 b/Python_Engine/Python/scripts/shared.ps1 new file mode 100644 index 00000000..cea9f6ea --- /dev/null +++ b/Python_Engine/Python/scripts/shared.ps1 @@ -0,0 +1,249 @@ +function Write-InfoMessage { + param ( + [Parameter(Mandatory = $true)] + [string]$Message, + [string]$FunctionName = $MyInvocation.MyCommand.Name + ) + + Write-Host "- [$FunctionName] Info: $Message" -ForegroundColor Cyan +} + +function Write-WarningMessage { + param ( + [Parameter(Mandatory = $true)] + [string]$Message, + [string]$FunctionName = $MyInvocation.MyCommand.Name + ) + + Write-Host "- [$FunctionName] Warning: $Message" -ForegroundColor Magenta +} + +function Write-ErrorMessage { + param ( + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$FunctionName = $MyInvocation.MyCommand.Name, + [switch]$ThrowError + ) + + Write-Host "m- [$FunctionName] Error: $Message" -ForegroundColor Red + if ($ThrowError) { + throw $Message + } +} + +function Read-UserInput { + param ( + [Parameter(Mandatory = $true)] + [string]$Message + ) + + Write-Host $Message -ForegroundColor Yellow -NoNewline + return Read-Host +} + +function Get-LocalPythonExecutable { + $functionName = $MyInvocation.MyCommand.Name + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $projectPython = Join-Path $workspaceRoot ".venv\Scripts\python.exe" + if (Test-Path $projectPython -PathType Leaf) { + Write-InfoMessage -Message "Using Python executable: $projectPython" -FunctionName $functionName + return $projectPython + } + else { + Write-ErrorMessage -Message "Local Python executable not found at $projectPython. Run the installation script first." -FunctionName $functionName -ThrowError + } +} + +function Get-TomlFile { + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $tomlFile = Join-Path $workspaceRoot "pyproject.toml" + if (Test-Path $tomlFile -PathType Leaf) { + return $tomlFile + } + else { + throw "pyproject.toml file not found at $tomlFile." + } +} + +function Register-Kernel { + param ( + [Parameter(Mandatory = $true)] + [string]$KernelName, + [Parameter(Mandatory = $false)] + [string]$DisplayName + ) + + $functionName = $MyInvocation.MyCommand.Name + + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $pythonExecutable = Get-LocalPythonExecutable + + if (-not $DisplayName) { + $DisplayName = "Python ($KernelName)" + } + + $locationPushed = $false + try { + Push-Location $workspaceRoot + $locationPushed = $true + Write-InfoMessage -Message "Creating $KernelName ($DisplayName) Jupyter kernel." -FunctionName $functionName + + & $pythonExecutable -m ipykernel install --user --name $KernelName --display-name $DisplayName + if ($LASTEXITCODE -ne 0) { + throw "Python failed to register the Jupyter kernel (exit code $LASTEXITCODE)." + } + } + catch { + Write-ErrorMessage -Message "Failed to create the $KernelName Jupyter kernel. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } + finally { + if ($locationPushed) { + Pop-Location + } + } +} + +function Unregister-Kernel { + param ( + [Parameter(Mandatory = $true)] + [string]$KernelName + ) + $functionName = $MyInvocation.MyCommand.Name + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $pythonExecutable = Get-LocalPythonExecutable + + try { + Push-Location $workspaceRoot + try { + Write-InfoMessage -Message "Removing $KernelName Jupyter kernel." -FunctionName $functionName + + & $pythonExecutable -m jupyter kernelspec uninstall $KernelName -f + $uninstallExitCode = $LASTEXITCODE + if ($uninstallExitCode -ne 0) { + Write-WarningMessage -Message "Jupyter could not remove the kernel (exit code $uninstallExitCode); checking its files directly." -FunctionName $functionName + } + + $kernelPath = Join-Path $env:APPDATA "jupyter\kernels\$KernelName" + if (Test-Path $kernelPath) { + Remove-PathIfExists -Path $kernelPath -Description "the $KernelName Jupyter kernel" -FunctionName $functionName + } + else { + Write-InfoMessage -Message "No remaining Jupyter kernel files found for $KernelName." -FunctionName $functionName + } + } + finally { + Pop-Location + } + } + catch { + Write-ErrorMessage -Message "Failed to remove the $KernelName Jupyter kernel. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } +} + +function Get-ToolkitName { + $tomlFile = Get-TomlFile + $tomlContent = Get-Content -LiteralPath $tomlFile -ErrorAction Stop + foreach ($line in $tomlContent) { + if ($line -match '^\s*name\s*=\s*"([^"]+)"\s*$') { + return $matches[1] + } + } + throw "Project name was not found in $tomlFile." +} + +function Remove-PathIfExists { + param ( + [Parameter(Mandatory = $true)] + [string]$Path, + [Parameter(Mandatory = $true)] + [string]$Description, + [Parameter(Mandatory = $true)] + [string]$FunctionName + ) + + if (Test-Path $Path) { + try { + Remove-Item -LiteralPath $Path -Recurse -Force -ErrorAction Stop + Write-InfoMessage -Message "Removed ${Description}: $Path" -FunctionName $FunctionName + } + catch { + Write-ErrorMessage -Message "Could not remove ${Description} at $Path. $($_.Exception.Message)" -FunctionName $FunctionName -ThrowError + } + } +} + +function Get-UVExecutable { + return Join-Path $env:ProgramData "BHoM\Extensions\Python\tools\uv.exe" +} + +function Install-UV { + $functionName = $MyInvocation.MyCommand.Name + $uvExecutable = Get-UVExecutable + $uvBinPath = Split-Path -Parent $uvExecutable + + if (-not (Test-Path -LiteralPath $uvExecutable -PathType Leaf)) { + try { + Write-InfoMessage -Message "Installing UV in $uvBinPath." -FunctionName $functionName + $escapedDirectory = $uvBinPath.Replace("'", "''") + & powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "`$ErrorActionPreference = 'Stop'; `$env:UV_INSTALL_DIR = '$escapedDirectory'; `$env:UV_NO_MODIFY_PATH = '1'; irm https://astral.sh/uv/install.ps1 | iex" + if ($LASTEXITCODE -ne 0) { + throw "UV installer exited with code $LASTEXITCODE." + } + } + catch { + Write-ErrorMessage -Message "Failed to install UV. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } + } + + try { + if (-not (Test-Path -LiteralPath $uvExecutable -PathType Leaf)) { + throw "The UV executable was not installed at $uvExecutable." + } + $otherPaths = @($env:Path -split ';' | Where-Object { $_.Trim().Trim('"') -ine $uvBinPath }) + $env:Path = (@($uvBinPath) + $otherPaths) -join ';' + Write-InfoMessage -Message "Using UV from $uvExecutable." -FunctionName $functionName + } + catch { + Write-ErrorMessage -Message "Could not locate UV. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } + + # install core tools + foreach ($tool in @("ruff", "jupyterlab", "sphinx")) { + try { + Write-InfoMessage -Message "Installing $tool to UV." -FunctionName $functionName + & $uvExecutable tool install $tool + if ($LASTEXITCODE -ne 0) { + throw "uv tool install exited with code $LASTEXITCODE." + } + } + catch { + Write-ErrorMessage -Message "Failed to install $tool to UV. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } + } +} + +function Sync-PythonEnvironment { + $functionName = $MyInvocation.MyCommand.Name + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $locationPushed = $false + + try { + Push-Location $workspaceRoot + $locationPushed = $true + Write-InfoMessage -Message "Installing Python packages with UV." -FunctionName $functionName + & (Get-UVExecutable) sync + if ($LASTEXITCODE -ne 0) { + throw "uv sync exited with code $LASTEXITCODE." + } + } + catch { + Write-ErrorMessage -Message "Failed to sync the Python environment. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } + finally { + if ($locationPushed) { + Pop-Location + } + } +} diff --git a/Python_Engine/Python/scripts/uninstall.ps1 b/Python_Engine/Python/scripts/uninstall.ps1 new file mode 100644 index 00000000..e2f6977e --- /dev/null +++ b/Python_Engine/Python/scripts/uninstall.ps1 @@ -0,0 +1,51 @@ +param ( + [switch]$Run +) + +. "$PSScriptRoot\shared.ps1" + +function Uninstall-Package { + $functionName = $MyInvocation.MyCommand.Name + + $workspaceRoot = Split-Path -Parent $PSScriptRoot + $locationPushed = $false + $toolkitName = "Python Toolkit" + $uninstallationComplete = $false + + try { + Push-Location $workspaceRoot + $locationPushed = $true + $toolkitName = Get-ToolkitName + + $userInput = Read-UserInput -Message "Do you really want to uninstall the $toolkitName Python package? (y/n): " + if ($userInput -ne "y") { + Write-InfoMessage -Message "Uninstallation aborted by the user." -FunctionName $functionName + return + } + + Write-InfoMessage -Message "Removing the Jupyter kernel." -FunctionName $functionName + Unregister-Kernel -KernelName $toolkitName + + Write-InfoMessage -Message "Removing the project virtual environment." -FunctionName $functionName + Remove-PathIfExists -Path (Join-Path $workspaceRoot ".venv") -Description "the $toolkitName virtual environment" -FunctionName $functionName + $uninstallationComplete = $true + } + catch { + Write-ErrorMessage -Message "Uninstallation failed for $toolkitName. $($_.Exception.Message)" -FunctionName $functionName -ThrowError + } + finally { + if ($locationPushed) { + Pop-Location + } + if ($uninstallationComplete) { + Write-InfoMessage -Message "Uninstallation process for $toolkitName completed successfully." -FunctionName $functionName + } + elseif ($userInput -eq "y") { + Write-WarningMessage -Message "Uninstallation process for $toolkitName did not complete." -FunctionName $functionName + } + } +} + +if ($Run -or $MyInvocation.InvocationName -ne '.') { + Uninstall-Package +} diff --git a/Python_Engine/Python/src/python_toolkit/bhom/assets/BHoM_Logo.png b/Python_Engine/Python/src/assets/BHoM_Logo.png similarity index 100% rename from Python_Engine/Python/src/python_toolkit/bhom/assets/BHoM_Logo.png rename to Python_Engine/Python/src/assets/BHoM_Logo.png diff --git a/Python_Engine/Python/src/python_toolkit/bhom/assets/bhom_icon.png b/Python_Engine/Python/src/assets/bhom_icon.png similarity index 100% rename from Python_Engine/Python/src/python_toolkit/bhom/assets/bhom_icon.png rename to Python_Engine/Python/src/assets/bhom_icon.png diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index ca4230df..20aabe05 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -9,6 +9,7 @@ BH.Engine.Python 10.0.0.0 ..\Build\ + Debug;Release;Test @@ -35,10 +36,13 @@ False + + + - + - + diff --git a/Python_Engine/Query/Directory.cs b/Python_Engine/Query/Directory.cs index e81943e2..17c22b62 100644 --- a/Python_Engine/Query/Directory.cs +++ b/Python_Engine/Query/Directory.cs @@ -21,7 +21,6 @@ */ using BH.oM.Base.Attributes; -using BH.oM.Python.Enums; using System.ComponentModel; using System.IO; @@ -36,47 +35,24 @@ public static string DirectoryExtensions() return Path.Combine(System.IO.Directory.GetParent(BH.Engine.Base.Query.BHoMFolder()).FullName, "Extensions"); } - [Description("The location where any BHoM-related Python kernels reside.")] - [Output("path", "The location where any BHoM-related Python kernels reside.")] - public static string DirectoryKernels() - { - - string dir = Path.Combine(System.Environment.GetFolderPath(System.Environment.SpecialFolder.ApplicationData), "jupyter", "kernels"); - if (!Directory.Exists(dir)) - Directory.CreateDirectory(dir); - return dir; - } - - [Description("The location where any BHoM-related Python code resides.")] - [Output("path", "The location where any BHoM-related Python code resides.")] - public static string DirectoryCode() - { - string dir = Path.Combine(Query.DirectoryExtensions(), "PythonCode"); - if (!Directory.Exists(dir)) - Directory.CreateDirectory(dir); - return dir; - } - [Description("The location where any BHoM-related Python environment (or virtualenv) resides.")] [Output("path", "The location where any BHoM-related Python environment (or virtualenv) resides.")] public static string DirectoryEnvironments() { - string dir = Path.Combine(Query.DirectoryExtensions(), "PythonEnvironments"); + string dir = Path.Combine(Query.DirectoryExtensions(), "Python"); if (!Directory.Exists(dir)) Directory.CreateDirectory(dir); return dir; } - [Description("The location where the base Python environment exists.")] - [Input("version", "The python version to get the base environment for.")] - [Output("path", "The location where the base Python environment exists.")] - public static string DirectoryBaseEnvironment(this PythonVersion version) + [Description("The location where any BHoM-related Python tools reside.")] + [Output("path", "The location where any BHoM-related Python tools reside.")] + public static string DirectoryTools() { - return Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName(), version.ToString()); + string dir = Path.Combine(Query.DirectoryExtensions(), "Python", "tools"); + if (!Directory.Exists(dir)) + Directory.CreateDirectory(dir); + return dir; } } } - - - - diff --git a/Python_Engine/Query/EmbeddableURL.cs b/Python_Engine/Query/EmbeddableURL.cs deleted file mode 100644 index 16cdee12..00000000 --- a/Python_Engine/Query/EmbeddableURL.cs +++ /dev/null @@ -1,97 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using BH.oM.Python.Enums; -using BH.oM.Base.Attributes; -using System.Collections.Generic; -using System.ComponentModel; - -namespace BH.Engine.Python -{ - public static partial class Query - { - [Description("Get the URL for the embeddable package containing the given version of Python.")] - [Input("version", "The version of Python to retrieve the URL.")] - [Output("url", "The url for the matching Python version.")] - public static string EmbeddableURL(this PythonVersion version) - { - // See the PythonVersion enum for why most of these are commented out. - Dictionary versions = new Dictionary() - { - //{ PythonVersion.v3_7_2, "https://www.python.org/ftp/python/3.7.2/python-3.7.2rc1-amd64.exe" }, - /*{ PythonVersion.v3_7_3, "https://www.python.org/ftp/python/3.7.3/python-3.7.3rc1-amd64.exe" }, - { PythonVersion.v3_7_4, "https://www.python.org/ftp/python/3.7.4/python-3.7.4rc2-amd64.exe" }, - { PythonVersion.v3_7_5, "https://www.python.org/ftp/python/3.7.5/python-3.7.5rc1-amd64.exe" }, - { PythonVersion.v3_7_6, "https://www.python.org/ftp/python/3.7.6/python-3.7.6rc1-amd64.exe" }, - { PythonVersion.v3_7_7, "https://www.python.org/ftp/python/3.7.7/python-3.7.7rc1-amd64.exe" }, - { PythonVersion.v3_7_8, "https://www.python.org/ftp/python/3.7.8/python-3.7.8rc1-amd64.exe" },*/ - { PythonVersion.v3_7, "https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe" }, - /*{ PythonVersion.v3_8_0, "https://www.python.org/ftp/python/3.8.0/python-3.8.0rc1-amd64.exe" }, - { PythonVersion.v3_8_1, "https://www.python.org/ftp/python/3.8.1/python-3.8.1rc1-amd64.exe" }, - { PythonVersion.v3_8_2, "https://www.python.org/ftp/python/3.8.2/python-3.8.2rc2-amd64.exe" }, - { PythonVersion.v3_8_3, "https://www.python.org/ftp/python/3.8.3/python-3.8.3rc1-amd64.exe" }, - { PythonVersion.v3_8_4, "https://www.python.org/ftp/python/3.8.4/python-3.8.4rc1-amd64.exe" }, - { PythonVersion.v3_8_5, "https://www.python.org/ftp/python/3.8.5/python-3.8.5-amd64.exe" }, - { PythonVersion.v3_8_6, "https://www.python.org/ftp/python/3.8.6/python-3.8.6rc1-amd64.exe" }, - { PythonVersion.v3_8_7, "https://www.python.org/ftp/python/3.8.7/python-3.8.7rc1-amd64.exe" }, - { PythonVersion.v3_8_8, "https://www.python.org/ftp/python/3.8.8/python-3.8.8rc1-amd64.exe" },*/ - { PythonVersion.v3_8, "https://www.python.org/ftp/python/3.8.9/python-3.8.9-amd64.exe" }, - /*{ PythonVersion.v3_9_0, "https://www.python.org/ftp/python/3.9.0/python-3.9.0rc2-amd64.exe" }, - { PythonVersion.v3_9_1, "https://www.python.org/ftp/python/3.9.1/python-3.9.1rc1-amd64.exe" }, - { PythonVersion.v3_9_2, "https://www.python.org/ftp/python/3.9.2/python-3.9.2rc1-amd64.exe" }, - { PythonVersion.v3_9_3, "https://www.python.org/ftp/python/3.9.3/python-3.9.3-amd64.exe" }, - { PythonVersion.v3_9_4, "https://www.python.org/ftp/python/3.9.4/python-3.9.4-amd64.exe" }, - { PythonVersion.v3_9_5, "https://www.python.org/ftp/python/3.9.5/python-3.9.5-amd64.exe" }, - { PythonVersion.v3_9_6, "https://www.python.org/ftp/python/3.9.6/python-3.9.6-amd64.exe" }, - { PythonVersion.v3_9_7, "https://www.python.org/ftp/python/3.9.7/python-3.9.7-amd64.exe" }, - { PythonVersion.v3_9_8, "https://www.python.org/ftp/python/3.9.8/python-3.9.8-amd64.exe" }, - { PythonVersion.v3_9_9, "https://www.python.org/ftp/python/3.9.9/python-3.9.9-amd64.exe" },*/ - { PythonVersion.v3_9, "https://www.python.org/ftp/python/3.9.10/python-3.9.10-amd64.exe" }, - /*{ PythonVersion.v3_10_0, "https://www.python.org/ftp/python/3.10.0/python-3.10.0rc2-amd64.exe" }, - { PythonVersion.v3_10_1, "https://www.python.org/ftp/python/3.10.1/python-3.10.1-amd64.exe" }, - { PythonVersion.v3_10_2, "https://www.python.org/ftp/python/3.10.2/python-3.10.2-amd64.exe" }, - { PythonVersion.v3_10_3, "https://www.python.org/ftp/python/3.10.3/python-3.10.3-amd64.exe" }, - { PythonVersion.v3_10_4, "https://www.python.org/ftp/python/3.10.4/python-3.10.4-amd64.exe" }, - { PythonVersion.v3_10_5, "https://www.python.org/ftp/python/3.10.5/python-3.10.5-amd64.exe" }, - { PythonVersion.v3_10_6, "https://www.python.org/ftp/python/3.10.6/python-3.10.6-amd64.exe" }, - { PythonVersion.v3_10_7, "https://www.python.org/ftp/python/3.10.7/python-3.10.7-amd64.exe" }, - { PythonVersion.v3_10_8, "https://www.python.org/ftp/python/3.10.8/python-3.10.8-amd64.exe" }, - { PythonVersion.v3_10_9, "https://www.python.org/ftp/python/3.10.9/python-3.10.9-amd64.exe" }, - { PythonVersion.v3_10_10, "https://www.python.org/ftp/python/3.10.10/python-3.10.10-amd64.exe" },*/ - { PythonVersion.v3_10, "https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe" }, - /*{ PythonVersion.v3_11_0, "https://www.python.org/ftp/python/3.11.0/python-3.11.0rc2-amd64.exe" }, - { PythonVersion.v3_11_1, "https://www.python.org/ftp/python/3.11.1/python-3.11.1-amd64.exe" }, - { PythonVersion.v3_11_2, "https://www.python.org/ftp/python/3.11.2/python-3.11.2-amd64.exe" }, - { PythonVersion.v3_11_3, "https://www.python.org/ftp/python/3.11.3/python-3.11.3-amd64.exe" },*/ - { PythonVersion.v3_11, "https://www.python.org/ftp/python/3.11.4/python-3.11.4-amd64.exe" }, - { PythonVersion.v3_12, "https://www.python.org/ftp/python/3.12.3/python-3.12.3-amd64.exe" }, - { PythonVersion.v3_13, "https://www.python.org/ftp/python/3.13.12/python-3.13.12-amd64.exe" }, - }; - - return versions[version]; - } - } -} - - - - diff --git a/Python_Engine/Query/IsValidEnvironmentName.cs b/Python_Engine/Query/IsValidEnvironmentName.cs deleted file mode 100644 index 16625a65..00000000 --- a/Python_Engine/Query/IsValidEnvironmentName.cs +++ /dev/null @@ -1,56 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using BH.oM.Python.Enums; -using BH.oM.Python; -using BH.oM.Base.Attributes; - -using System.ComponentModel; -using System.IO; -using System.Linq; -using System; -using System.Collections.Generic; - -namespace BH.Engine.Python -{ - public static partial class Query - { - [Description("Check whether a string is valid as BHoM Python Environment name.")] - [Input("name", "The name given to the BHoM Python Environment.")] - [Output("valid", "True if valid, False if not.")] - public static bool IsValidEnvironmentName(string name) - { - List invalidChars = new List() { ' ' }; - invalidChars.AddRange(Path.GetInvalidPathChars().ToList()); - invalidChars.AddRange(Path.GetInvalidFileNameChars().ToList()); - if (name.Any(x => invalidChars.Contains(x))) - { - return false; - } - return true; - } - } -} - - - - diff --git a/Python_Engine/Query/ToolkitName.cs b/Python_Engine/Query/ToolkitName.cs index c0088f6b..e8e3ba38 100644 --- a/Python_Engine/Query/ToolkitName.cs +++ b/Python_Engine/Query/ToolkitName.cs @@ -32,7 +32,7 @@ public static partial class Query [Output("name", "The name of this toolkit.")] public static string ToolkitName() { - return "Python_Toolkit"; + return "python_toolkit"; } } } diff --git a/Python_Engine/Query/UVExecutable.cs b/Python_Engine/Query/UVExecutable.cs new file mode 100644 index 00000000..e1a87ef5 --- /dev/null +++ b/Python_Engine/Query/UVExecutable.cs @@ -0,0 +1,38 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using BH.oM.Base.Attributes; +using System.ComponentModel; +using System.IO; + +namespace BH.Engine.Python +{ + public static partial class Query + { + [Description("The BHoM-managed UV executable. UV installations on PATH are not used.")] + [Output("executable", "The path to uv.exe in the BHoM tools directory.")] + public static string UVExecutable() + { + return Path.Combine(DirectoryTools(), "uv.exe"); + } + } +} diff --git a/Python_Engine/Query/Version.cs b/Python_Engine/Query/Version.cs index 7e2db22d..244e6eee 100644 --- a/Python_Engine/Query/Version.cs +++ b/Python_Engine/Query/Version.cs @@ -21,13 +21,9 @@ */ using BH.oM.Base.Attributes; -using BH.oM.Python.Enums; -using System; -using System.Collections.Generic; +using BH.oM.Python; using System.ComponentModel; -using System.Diagnostics; using System.IO; -using System.Linq; namespace BH.Engine.Python { @@ -36,54 +32,27 @@ public static partial class Query [Description("Gets the Python version for the requested Python executable.")] [Input("pythonExecutable", "Path to python.exe.")] [Output("version", "Python version of the requested Python executable.")] - public static PythonVersion Version(string pythonExecutable) + public static string Version(string pythonExecutable) { - if (!File.Exists(pythonExecutable)) + if (!File.Exists(pythonExecutable)) { - return PythonVersion.Undefined; + BH.Engine.Base.Compute.RecordError(new FileNotFoundException(), $"No file found at {pythonExecutable}"); + return null; } - Process process = new Process() - { - StartInfo = new ProcessStartInfo() - { - FileName = pythonExecutable, - Arguments = $"--version", - RedirectStandardError = true, - RedirectStandardOutput = true, - UseShellExecute = false, - } - }; - - try - { - string versionString; - using (Process p = Process.Start(process.StartInfo)) - { - string standardError = p.StandardError.ReadToEnd(); - versionString = p.StandardOutput.ReadToEnd().TrimEnd(); - p.WaitForExit(); - if (p.ExitCode != 0) - { - BH.Engine.Base.Compute.RecordError($"Error getting Python version.\n{standardError}"); - return PythonVersion.Undefined; - } - } + var result = BH.Engine.Python.Compute.RunCommandStdout(command: $"\"{pythonExecutable}\" --version"); + if (result.Item3 != 0) + return null; - List strings = versionString.Replace("Python ", "").Replace(".", "_").Split('_').ToList(); - strings.RemoveAt(strings.Count - 1); - string enumParseable = "v" + strings.Aggregate((a, b) => $"{a}_{b}"); - - return (PythonVersion)Enum.Parse(typeof(PythonVersion), enumParseable); - } - catch - { - return PythonVersion.Undefined; - } + return result.Item1.Replace("Python ", ""); + } + [Description("Get the Python version for the installed PythonEnvironment.")] + [Input("venv", "The PythonEnvironment.")] + [Output("version", "Python version of the installed PythonEnvironment.")] + public static string Version(this PythonEnvironment venv) + { + return Version(venv.Executable); } } } - - - diff --git a/Python_Engine/Query/VirtualEnvironment.cs b/Python_Engine/Query/VirtualEnvironment.cs index f20f0e19..f2b64874 100644 --- a/Python_Engine/Query/VirtualEnvironment.cs +++ b/Python_Engine/Query/VirtualEnvironment.cs @@ -20,82 +20,74 @@ * along with this code. If not, see . */ -using BH.oM.Python.Enums; using BH.oM.Python; using BH.oM.Base.Attributes; using System.ComponentModel; using System.IO; -using System.Linq; -using System; -using System.Collections.Generic; namespace BH.Engine.Python { public static partial class Query { - [Description("Check whether a named BHoM Python virtual environment exists.")] - [Input("envName", "The name given to the BHoM Python Environment.")] - [Input("pythonVersion", "The version to check against.")] - [Output("exists", "True if exists, False if not.")] - public static bool VirtualEnvironmentExists(string envName, PythonVersion pythonVersion = PythonVersion.Undefined) + [Description("Get the path to the named Python virtual environment executable.")] + [Input("name", "The name given to the Python Environment.")] + [Output("executable", "The path to the executable.")] + public static string VirtualEnvironmentExecutable(string name) { - string executable = VirtualEnvironmentExecutable(envName); - bool directoryExists = Directory.Exists(VirtualEnvironmentDirectory(envName)); - bool executableExists = File.Exists(executable); - bool kernelExists = Directory.Exists(VirtualEnvironmentKernel(envName)); - bool versionMatches = false; - - if (executableExists) - versionMatches = Version(executable) == pythonVersion; + string exePath = Path.Combine(Query.DirectoryEnvironments(), name, ".venv", "Scripts", "python.exe"); + if (!File.Exists(exePath)) + { + Base.Compute.RecordError($"{exePath} was not found."); + return null; + } - if (pythonVersion == PythonVersion.Undefined) - return directoryExists && executableExists && kernelExists; - - return directoryExists && executableExists && kernelExists && versionMatches; + return exePath; } - [Description("Get the path to the named BHoM Python virtual environment kernel.")] - [Input("envName", "The name given to the BHoM Python Environment.")] - [Output("kernelDirectory", "The path to the kernel directory.")] - public static string VirtualEnvironmentKernel(string envName) + [Description("Check whether a named Python virtual environment exists.")] + [Input("name", "The name given to the Python Environment.")] + [Output("exists", "True if exists, False if not.")] + public static bool VirtualEnvironmentExists(string name) { - return Path.Combine(Query.DirectoryKernels(), envName.ToLower()); - } + string expectedVenvPath = Path.Combine(Query.DirectoryEnvironments(), name, ".venv"); - [Description("Get the path to the named BHoM Python virtual environment executable.")] - [Input("envName", "The name given to the BHoM Python Environment.")] - [Output("executable", "The path to the executable.")] - public static string VirtualEnvironmentExecutable(string envName) - { - return Path.Combine(Query.DirectoryEnvironments(), envName, "Scripts", "python.exe"); - } + if (Directory.Exists(expectedVenvPath) && File.Exists(Path.Combine(expectedVenvPath, "Scripts", "python.exe"))) + return true; - [Description("Get the path to the named BHoM Python virtual environment.")] - [Input("envName", "The name given to the BHoM Python Environment.")] - [Output("directory", "The directory where the virtual environment is located.")] - public static string VirtualEnvironmentDirectory(string envName) - { - return Path.Combine(Query.DirectoryEnvironments(), envName); + return false; } - [Description("Get the named BHoM Python virtual environment.")] - [Input("envName", "The name given to the BHoM Python Environment.")] - [Output("environment", "The BHoM Python Environment.")] - public static PythonEnvironment VirtualEnvironment(string envName) + [Description("Get the named Python virtual environment.")] + [Input("name", "The name given to the Python Environment.")] + [Output("environment", "The Python Environment.")] + public static PythonEnvironment VirtualEnvironment(string name) { - if (!VirtualEnvironmentExists(envName)) + if (!VirtualEnvironmentExists(name)) + { + Base.Compute.RecordError($"{name} was not found in {Query.DirectoryEnvironments()}"); return null; - + } + return new PythonEnvironment() { - Name = envName, - Executable = VirtualEnvironmentExecutable(envName), + Name = name, + Executable = VirtualEnvironmentExecutable(name), }; } - } -} - - + [Description("Get a named Python virtual environment.")] + [Input("name", "The name given to the Python Environment.")] + [Output("directory", "The directory in which the venv resides.")] + public static string VirtualEnvironmentDirectory(string name) + { + if (!VirtualEnvironmentExists(name)) + { + Base.Compute.RecordError($"{name} was not found in {Query.DirectoryEnvironments()}"); + return null; + } + return Path.Combine(Query.DirectoryEnvironments(), name); + } + } +} diff --git a/Python_Toolkit.sln b/Python_Toolkit.sln index 31fdc786..9f41590f 100644 --- a/Python_Toolkit.sln +++ b/Python_Toolkit.sln @@ -7,6 +7,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python_oM", "Python_oM\Pyth EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python_Engine", "Python_Engine\Python_Engine.csproj", "{D24A2970-618D-4149-B123-99F406D867D6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python_Engine_Test", ".ci\unit_tests\Python_Engine_Test\Python_Engine_Test.csproj", "{E654D8C0-C9A3-C0D9-F687-4A0CA64CA548}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -21,6 +23,10 @@ Global {D24A2970-618D-4149-B123-99F406D867D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {D24A2970-618D-4149-B123-99F406D867D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {D24A2970-618D-4149-B123-99F406D867D6}.Release|Any CPU.Build.0 = Release|Any CPU + {E654D8C0-C9A3-C0D9-F687-4A0CA64CA548}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E654D8C0-C9A3-C0D9-F687-4A0CA64CA548}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E654D8C0-C9A3-C0D9-F687-4A0CA64CA548}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E654D8C0-C9A3-C0D9-F687-4A0CA64CA548}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Python_oM/Enums/PythonVersion.cs b/Python_oM/Enums/PythonVersion.cs deleted file mode 100644 index 301febe1..00000000 --- a/Python_oM/Enums/PythonVersion.cs +++ /dev/null @@ -1,137 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2026, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using System.ComponentModel; - -namespace BH.oM.Python.Enums -{ - // There are multiple specific versions here that are commented out - When installing, only one specific version (e.g 3.10.x) per minor python version (e.g 3.x) can be installed, and changes between specific versions are usually bug fixes and not breaking. however the specific versions are kept commented for posterity. - public enum PythonVersion - { - [Description("Undefined")] - Undefined, - /*[Description("3.7.0")] - v3_7_0, - [Description("3.7.1")] - v3_7_1, - [Description("3.7.2")] - v3_7_2, - [Description("3.7.3")] - v3_7_3, - [Description("3.7.4")] - v3_7_4, - [Description("3.7.5")] - v3_7_5, - [Description("3.7.6")] - v3_7_6, - [Description("3.7.7")] - v3_7_7, - [Description("3.7.8")] - v3_7_8,*/ - [Description("3.7")] //3.7.9 - v3_7, - /*[Description("3.8.0")] - v3_8_0, - [Description("3.8.1")] - v3_8_1, - [Description("3.8.2")] - v3_8_2, - [Description("3.8.3")] - v3_8_3, - [Description("3.8.4")] - v3_8_4, - [Description("3.8.5")] - v3_8_5, - [Description("3.8.6")] - v3_8_6, - [Description("3.8.7")] - v3_8_7, - [Description("3.8.8")] - v3_8_8,*/ - [Description("3.8")] //3.8.9 - v3_8, - //[Description("3.8.10")] - //v3_8_10, - /*[Description("3.9.0")] - v3_9_0, - [Description("3.9.1")] - v3_9_1, - [Description("3.9.2")] - v3_9_2, - [Description("3.9.3")] - v3_9_3, - [Description("3.9.4")] - v3_9_4, - [Description("3.9.5")] - v3_9_5, - [Description("3.9.6")] - v3_9_6, - [Description("3.9.7")] - v3_9_7, - [Description("3.9.8")] - v3_9_8, - [Description("3.9.9")] - v3_9_9,*/ - [Description("3.9")] //3.9.10 - v3_9, - /*[Description("3.10.0")] - v3_10_0, - [Description("3.10.1")] - v3_10_1, - [Description("3.10.2")] - v3_10_2, - [Description("3.10.3")] - v3_10_3, - [Description("3.10.4")] - v3_10_4, - [Description("3.10.5")] - v3_10_5, - [Description("3.10.6")] - v3_10_6, - [Description("3.10.7")] - v3_10_7, - [Description("3.10.8")] - v3_10_8, - [Description("3.10.9")] - v3_10_9, - [Description("3.10.10")] - v3_10_10,*/ - [Description("3.10")] //3.10.11 - v3_10, - /*[Description("3.11.0")] - v3_11_0, - [Description("3.11.1")] - v3_11_1, - [Description("3.11.2")] - v3_11_2, - [Description("3.11.3")] - v3_11_3,*/ - [Description("3.11")] //3.11.4 - v3_11, - [Description("3.12")] - v3_12, - [Description("3.13")] - v3_13, - } -} - - diff --git a/Python_oM/PythonEnvironment.cs b/Python_oM/PythonEnvironment.cs index 7740cb3f..7f81cb67 100644 --- a/Python_oM/PythonEnvironment.cs +++ b/Python_oM/PythonEnvironment.cs @@ -22,8 +22,6 @@ using BH.oM.Base; -using BH.oM.Python.Enums; -using System.Collections.Generic; using System.ComponentModel; namespace BH.oM.Python @@ -37,8 +35,3 @@ public class PythonEnvironment : IObject public virtual string Executable { get; set; } = string.Empty; } } - - - - - diff --git a/Python_oM/Python_oM.csproj b/Python_oM/Python_oM.csproj index c9a7e568..d28aace7 100644 --- a/Python_oM/Python_oM.csproj +++ b/Python_oM/Python_oM.csproj @@ -10,6 +10,7 @@ BH.oM.Python 10.0.0.0 ..\Build\ + Debug;Release;Test @@ -21,7 +22,7 @@ - + diff --git a/README.md b/README.md index 7018811d..318a646d 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,6 @@ There are two main events to install the toolkit: - Install the python bindings of the Python_Toolkit from C:\ProgramData\BHoM\Extensions\Python\src This contains all the code that is currently developed in the toolkit in python - ## Building the BHoM and the Toolkits from Source ## You will need the following to build BHoM: @@ -41,6 +40,13 @@ You will need the following to build BHoM: - Microsoft .NET Framework 4.0 and above (included with Visual Studio 2013) - Note that there are no software - specific dependencies (only operating system relevant), this is specific: BHoM is a software agnostic object model. +## Running commands and creating environments + +UV is installed at `Path.Combine(Query.DirectoryTools(), "uv.exe")`. Engine methods, tests, and PowerShell scripts invoke this BHoM-managed executable directly, never an unrelated UV on PATH. Installation prepends the tools directory to the current process PATH without changing the user or system PATH. + +`RunCommandStdout` returns `BH.oM.Base.Output`: `Item1` is stdout, `Item2` is stderr, and `Item3` is the exit code (zero for success, -1 for timeout). Both streams are retained on success, failure, and timeout. Callers that previously consumed a string must select the appropriate output. Command failures are also recorded as BHoM errors; invalid working directories and process-start failures still throw. + +Creating an environment from a TOML file updates its static `[project].version` to `BH.Engine.Base.Query.BHoMVersion()` before installation. Other version fields are left unchanged. Recreating that environment preserves the TOML file and package sources. ## Want to contribute? ## diff --git a/Versioning_93.json b/Versioning_93.json new file mode 100644 index 00000000..e2425a36 --- /dev/null +++ b/Versioning_93.json @@ -0,0 +1,47 @@ +{ + "Namespace": { + "ToNew": {}, + "ToOld": {} + }, + "Type": { + "ToNew": {}, + "ToOld": {} + }, + "Property": { + "ToNew": {}, + "ToOld": {} + }, + "MessageForDeleted": { + "BH.Engine.Python.Compute.BasePythonEnvironment(BH.oM.Python.Enums.PythonVersion, System.Boolean, System.Boolean)": "Python environments are now managed by UV. Use InstallPythonToolkitVenv to install the toolkit environment, or VirtualEnvironment with a Python version string to create a named environment. Reconnect inputs manually.", + "BH.Engine.Python.Compute.DownloadFile(System.String, System.String, System.String, System.Boolean)": "This download helper has been removed. Python installation is now handled by UV. For other downloads, use a separate file-download workflow.", + "BH.Engine.Python.Compute.DownloadPythonVersion(BH.oM.Python.Enums.PythonVersion)": "Python installation is now handled by UV. Use VirtualEnvironment with a Python version string, such as \"3.10\", instead.", + "BH.Engine.Python.Compute.InstallPackages(BH.oM.Python.PythonEnvironment, System.Collections.Generic.List)": "Replace this component with InstallPackages(PythonEnvironment, List, Boolean). Supply an environment created using the UV workflow and set run to true.", + "BH.Engine.Python.Compute.InstallPackages(System.String, System.Collections.Generic.List)": "Installation using an executable path has been removed. Use InstallPackages with a PythonEnvironment created using the UV workflow and set run to true.", + "BH.Engine.Python.Compute.InstallRequirements(BH.oM.Python.PythonEnvironment, System.String)": "Requirements-file installation has been removed. Declare dependencies in pyproject.toml and create the environment from that file, or supply package requirements to InstallPackages with run set to true.", + "BH.Engine.Python.Compute.InstallRequirements(System.String, System.String)": "Requirements-file installation using an executable path has been removed. Declare dependencies in pyproject.toml and create the environment from that file, or use InstallPackages with a PythonEnvironment and run set to true.", + "BH.Engine.Python.Compute.InstallPackageLocal(BH.oM.Python.PythonEnvironment, System.String)": "Replace this component with InstallPackageLocal(PythonEnvironment, String, Boolean). Supply an environment created using the UV workflow and set run to true.", + "BH.Engine.Python.Compute.InstallPackageLocal(System.String, System.String)": "Local package installation using an executable path has been removed. Use InstallPackageLocal with a PythonEnvironment created using the UV workflow and set run to true.", + "BH.Engine.Python.Compute.RemoveKernel(System.String)": "Use RemoveJupyterKernel with the name of a UV-managed BHoM environment and set run to true. Legacy kernels may require manual removal using Jupyter.", + "BH.Engine.Python.Compute.RemoveAllKernels()": "Bulk kernel removal has been removed. Use RemoveJupyterKernel for individual UV-managed BHoM environments with run set to true. Legacy kernels may require manual removal using Jupyter.", + "BH.Engine.Python.Compute.RemoveVirtualEnvironment(System.String)": "Replace this component with RemoveVirtualEnvironment(String, Boolean) and set run to true. It removes environments in the new BHoM Python directory; legacy environments require separate cleanup.", + "BH.Engine.Python.Compute.RemoveAllVirtualEnvironments()": "Bulk environment removal has been removed. Use RemoveVirtualEnvironment for each named environment and set run to true. RemoveEverything also removes BHoM-managed UV tools, so use it only when that broader cleanup is intended.", + "BH.Engine.Python.Compute.RemoveBaseVersion(BH.oM.Python.Enums.PythonVersion)": "The former base Python installation workflow has been removed. Use RemoveVirtualEnvironment for named BHoM environments. Removing UV does not uninstall UV-managed Python runtimes or legacy Python installations.", + "BH.Engine.Python.Compute.RemoveEverything()": "Replace this component with RemoveEverything(Boolean) and set run to true. Review its scope: it targets the new BHoM Python environments and managed UV tools, not legacy installations.", + "BH.Engine.Python.Compute.RequirementsTxt(System.String, System.String)": "Requirements-file generation has been removed. Manage project dependencies using pyproject.toml and UV. Use UV export separately if a requirements.txt file is needed.", + "BH.Engine.Python.Compute.UnzipFile(System.String, System.String, System.String, System.Boolean)": "This archive-extraction helper has been removed. Use a separate archive-extraction workflow if needed.", + "BH.Engine.Python.Compute.UpdateBHoMPackages(System.String, BH.oM.Python.Enums.PythonVersion)": "Use UpdateBHoMPackages(PythonEnvironment) with an environment created using the UV workflow. The PythonVersion enum input is no longer supported.", + "BH.Engine.Python.Compute.VirtualEnvironment(BH.oM.Python.Enums.PythonVersion, System.String, System.Boolean)": "Replace this component with VirtualEnvironment(String, String, Boolean). Replace the PythonVersion enum with a version string, such as \"3.10\", and recreate the environment using UV.", + "BH.Engine.Python.Query.DirectoryKernels()": "The fixed kernel-directory query has been removed. Use Jupyter kernelspec discovery to locate registered kernels.", + "BH.Engine.Python.Query.DirectoryCode()": "The separate PythonCode directory is no longer used. Toolkit project files now reside under DirectoryEnvironments(), within the toolkit's named directory.", + "BH.Engine.Python.Query.DirectoryBaseEnvironment(BH.oM.Python.Enums.PythonVersion)": "Base Python environments are no longer organised by PythonVersion. Use a named UV-managed environment and query its directory or executable instead.", + "BH.Engine.Python.Query.EmbeddableURL(BH.oM.Python.Enums.PythonVersion)": "Python downloads are now managed by UV. Use VirtualEnvironment with a Python version string instead of retrieving an installer URL.", + "BH.Engine.Python.Query.IsValidEnvironmentName(System.String)": "This legacy environment-name validation method has been removed. Review names against the requirements of the UV environment and Jupyter kernel workflows.", + "BH.Engine.Python.Query.VirtualEnvironmentExists(System.String, BH.oM.Python.Enums.PythonVersion)": "Replace this component with VirtualEnvironmentExists(String). It checks the new UV-managed BHoM environment layout and no longer accepts a PythonVersion input.", + "BH.Engine.Python.Query.VirtualEnvironmentKernel(System.String)": "This kernel-path query has been removed. Use Jupyter kernelspec discovery to locate a registered kernel.", + "BH.oM.Python.Enums.PythonVersion": "Python versions are now specified as strings, such as \"3.10\" or \"3.11\". Replace enum inputs with the appropriate version string and recreate environments using the UV workflow." + }, + "MessageForNoUpgrade": { + "BH.Engine.Python.Compute.RunCommandStdout(System.String, System.Boolean, System.String, System.Double)": "The output has changed from a single string to stdout, stderr and exit code. Review and reconnect downstream components manually. Exit code zero indicates success; -1 indicates timeout.", + "BH.Engine.Python.Query.Version(System.String)": "The output is now a Python version string instead of PythonVersion. Replace downstream enum-based comparisons and inputs with string-based equivalents." + } +} \ No newline at end of file