Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@ public static class OutputWriter
public static int Write(string packageRoot, IEnumerable<RenderedType> rendered, bool cleanRedundant, Reporter reporter, ISet<string> cleanAssemblies = null)
{
Dictionary<string, string> fileTextByPath = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
Dictionary<string, string> typeByPath = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
List<string> clashes = new List<string>();
foreach (RenderedType type in rendered)
{
string relativePath = type.Ref.RelativePath + ".py";
if (fileTextByPath.ContainsKey(relativePath))
{
reporter.Warn(type.Model.ClrName, $"output path {relativePath} is already used by another type, skipped");
clashes.Add($"{typeByPath[relativePath]} and {type.Model.ClrName} would both be written to {relativePath}");
continue;
}

fileTextByPath[relativePath] = type.Text;
typeByPath[relativePath] = type.Model.ClrName;
string[] segments = relativePath.Split('/');

for (int i = 1; i < segments.Length; i++)
Expand All @@ -38,6 +41,10 @@ public static int Write(string packageRoot, IEnumerable<RenderedType> rendered,
}
}

//File names ignore upper and lower case on Windows, so one of the two classes would silently be lost
if (clashes.Count > 0)
throw new InvalidOperationException("Two classes would be written to the same file:" + Environment.NewLine + string.Join(Environment.NewLine, clashes));

if (cleanRedundant)
CleanRedundantExistingFiles(packageRoot, fileTextByPath, cleanAssemblies);

Expand All @@ -60,6 +67,56 @@ public static int Write(string packageRoot, IEnumerable<RenderedType> rendered,

/***************************************************/

[Description("Lists the assembly folders of the package that this run did not generate: folders of the package root, other than the support and hidden ones, that " +
"no rendered type belongs to. With repos (a run for some repositories) only the folders whose Python files name one of those repositories count, so the " +
"folders of other repositories are left alone. Generator calls it after a full run, so a failed build of an assembly is noticed instead of leaving its old files behind.")]
public static List<string> MissingAssemblyFolders(string packageRoot, IEnumerable<RenderedType> rendered, IEnumerable<string> repos)
{
HashSet<string> generated = new HashSet<string>(rendered.Select(r => r.Ref.RelativePath.Split('/')[0]), StringComparer.OrdinalIgnoreCase);
List<string> missing = new List<string>();
if (!Directory.Exists(packageRoot))
return missing;

foreach (string folder in Directory.GetDirectories(packageRoot).Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal))
{
if (folder.StartsWith("_") || folder.StartsWith(".") || generated.Contains(folder))
continue;

if (repos != null && !repos.Contains(RepoOfFolder(Path.Combine(packageRoot, folder)), StringComparer.OrdinalIgnoreCase))
continue;

missing.Add(folder);
}

return missing;
}

/***************************************************/

[Description("Reads the repository that the generated Python files of a folder name in their marker, or null when no file has one. MissingAssemblyFolders uses it to " +
"tell which repository a folder that was not generated belongs to.")]
private static string RepoOfFolder(string folder)
{
const string key = "'repo': '";
foreach (string file in Directory.EnumerateFiles(folder, "*.py", SearchOption.AllDirectories))
{
foreach (string line in File.ReadLines(file, m_Utf8))
{
int start = line.IndexOf(key, StringComparison.Ordinal);
if (start < 0)
continue;

start += key.Length;
int end = line.IndexOf('\'', start);
return end < 0 ? null : line.Substring(start, end - start);
}
}

return null;
}

/***************************************************/

[Description("Deletes .py files in the folders of the written assemblies that are no longer generated, then removes the empty folders. Write calls it when "
+ "stale output must be cleaned; cleanAssemblies limits it to the assemblies of the requested repositories.")]
private static void CleanRedundantExistingFiles(string packageRoot, Dictionary<string, string> fileTextByPath, ISet<string> cleanAssemblies)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ private string Assemble(TypeModel model, string classLine, string body, NameTrac
{
StringBuilder sb = new StringBuilder();
sb.Append($"# Generated by BHoM_PythonSchema from the C# oM type {model.ClrName}. Do not edit by hand.\n");
sb.Append("# Copyright (c) 2015 - 2026, the respective contributors. All rights reserved.\n");
sb.Append("# Copyright (c) the respective contributors. All rights reserved.\n");
sb.Append("# Licensed under the GNU Lesser General Public License, version 3.0 or later.\n");

bool isModel = model.Kind != TypeKind.Enum && model.Kind != TypeKind.Interface;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ public class GenerationOptions

public string ReleaseVersion; //Major.Minor of a release (for example 10.0): _version.py then holds the release version, and the run fails when it is not the version read

public bool CleanMissingAssemblies; //delete the assembly folders of the package that a full run did not generate, instead of only listing them

public string ManifestPath;

public string ReportPath;
Expand Down
24 changes: 23 additions & 1 deletion .ci/generation/PythonSchemaGeneration.Core/Generator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,12 @@ public static GenerationResult Run(GenerationOptions options)
{
result.FilesWritten = OutputWriter.Write(options.OutputFolder, result.Types, options.TypeFilter == null, reporter, repoAssemblies);

//A partial run (--types) does not say what the package is, so it leaves the version file alone
//A partial run (--types) does not say what the package is, so it leaves the version file and the other folders alone
if (options.TypeFilter == null)
{
result.FilesWritten += WriteVersionFile(options.OutputFolder, result.BHoMVersion, options.ReleaseVersion != null, reporter);
HandleMissingAssemblyFolders(options, result, reporter);
}
}

if (options.ManifestPath != null)
Expand All @@ -122,6 +125,25 @@ public static GenerationResult Run(GenerationOptions options)

/***************************************************/

[Description("Warns about every assembly folder of the package that this full run did not generate (for example because its build failed), or deletes it with " +
"--clean-missing-assemblies. Run calls it after the classes are written; the warning is a new kind for the baseline, so the weekly run fails loudly.")]
private static void HandleMissingAssemblyFolders(GenerationOptions options, GenerationResult result, Reporter reporter)
{
foreach (string folder in OutputWriter.MissingAssemblyFolders(options.OutputFolder, result.Types, options.Repos))
{
if (options.CleanMissingAssemblies)
{
Directory.Delete(Path.Combine(options.OutputFolder, folder), true);
reporter.Info($"Deleted the assembly folder {folder}, which this run did not generate");
result.FilesWritten++;
}
else
reporter.Warn($"assembly folder {folder} was not generated in this run, its old Python files are kept (use --clean-missing-assemblies to delete them)");
}
}

/***************************************************/

[Description("Writes bhom_schema/_version.py with the BHoM version the classes come from and the package version: that plus .0 for a release, and that plus .0.dev0 " +
"for develop, so the weekly run changes the file only when the BHoM version changes. Returns the number of files written (0 when it is unchanged). Run calls it " +
"after the classes are written; setuptools reads the package version from this file.")]
Expand Down
14 changes: 12 additions & 2 deletions .ci/generation/PythonSchemaGeneration/CommandLine.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,13 @@ public class CommandLine
{
/***************************************************/

private static readonly string[] m_Known = { "output", "assemblies", "org", "repos", "types", "manifest", "report", "baseline-warnings", "write-baseline", "release" };
private static readonly string[] m_Known = { "output", "assemblies", "org", "repos", "types", "manifest", "report", "baseline-warnings", "write-baseline", "release", "clean-missing-assemblies" };

//Options that stand alone, without a value
private static readonly string[] m_Flags = { "clean-missing-assemblies" };

public const string Usage = "Usage: PythonSchemaGeneration --output <bhom_schema folder> [--assemblies <folder>] [--org BHoM,Other] [--repos BHoM/BHoM,BHoM/Other_Toolkit] "
+ "[--types A,B] [--manifest <file>] [--report <file>] [--baseline-warnings <file>] [--write-baseline <file>] [--release <Major.Minor>]";
+ "[--types A,B] [--manifest <file>] [--report <file>] [--baseline-warnings <file>] [--write-baseline <file>] [--release <Major.Minor>] [--clean-missing-assemblies]";

public GenerationOptions Options { get; private set; }

Expand Down Expand Up @@ -39,6 +42,12 @@ public static CommandLine Parse(string[] args)
if (!m_Known.Contains(name))
return result.Fail($"Unknown option --{name}.");

if (m_Flags.Contains(name))
{
arguments[name] = "true";
continue;
}

if (i + 1 >= args.Length || args[i + 1].StartsWith("--"))
return result.Fail($"Option --{name} needs a value.");

Expand Down Expand Up @@ -66,6 +75,7 @@ public static CommandLine Parse(string[] args)
result.Options.TypeFilter = Split(types);

result.Options.ReleaseVersion = arguments.GetValueOrDefault("release");
result.Options.CleanMissingAssemblies = arguments.ContainsKey("clean-missing-assemblies");
result.BaselinePath = arguments.GetValueOrDefault("baseline-warnings");
result.WriteBaselinePath = arguments.GetValueOrDefault("write-baseline");
return result;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public class Scalars : BHoMObject
{
public virtual DateTime Created { get; set; } = DateTime.Now;
public virtual DateTime Fixed { get; set; } = new DateTime(2020, 1, 2, 3, 4, 5);
public virtual DateTimeOffset Stamp { get; set; } = new DateTimeOffset(2020, 1, 2, 3, 4, 5, TimeSpan.Zero);
public virtual Guid Key { get; set; } = Guid.Empty;
public virtual decimal Cost { get; set; } = 1.5m;
public virtual Color Colour { get; set; } = Color.Empty;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,18 @@ public void KnownOptionsAreRead()

/***************************************************/

[Test]
[Description("Checks that --clean-missing-assemblies is a flag that needs no value, may stand anywhere, and is off when it is not given. Protects the option that " +
"deletes the assembly folders a full run did not generate.")]
public void CleanMissingAssembliesIsAFlag()
{
Assert.That(CommandLine.Parse(new[] { "--output", "out" }).Options.CleanMissingAssemblies, Is.False);
Assert.That(CommandLine.Parse(new[] { "--output", "out", "--clean-missing-assemblies" }).Options.CleanMissingAssemblies, Is.True);
Assert.That(CommandLine.Parse(new[] { "--clean-missing-assemblies", "--output", "out", "--repos", "BHoM/A" }).Options.Repos, Is.EqualTo(new[] { "BHoM/A" }));
}

/***************************************************/

[TestCase(new[] { "--output", "o", "--repo", "BHoM/A" }, "Unknown option --repo")]
[TestCase(new[] { "--output", "o", "--types" }, "--types needs a value")]
[TestCase(new[] { "--output", "--types", "A" }, "--output needs a value")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ public void VolatileAndScalarDefaults()
string text = R(typeof(Scalars)).Text;
Assert.That(text, Does.Contain("Created: datetime = Field(default_factory=datetime.now)"));
Assert.That(text, Does.Contain("Fixed: datetime = datetime(2020, 1, 2, 3, 4, 5, 0)"));
Assert.That(text, Does.Contain("Stamp: Annotated[datetime, ClrType('System.DateTimeOffset')]"), "a DateTimeOffset keeps its C# type in the ClrType marker");
Assert.That(text, Does.Contain("Key: UUID = UUID(int=0)"));
Assert.That(text, Does.Contain("Cost: Decimal = Decimal('1.5')"));
Assert.That(text, Does.Contain("Colour: Color = Field(default_factory=Color)"));
Expand Down Expand Up @@ -382,5 +383,17 @@ public void CyclicReferencesUseBottomImports()
Assert.That(R(typeof(PointLike)).Text, Does.Not.Contain("model_rebuild"), "types outside a cycle do not need a rebuild");
Assert.That(FixtureOutput.Result.CyclicTypes, Is.EqualTo(2));
}

[Test]
[Description("Checks that the header of every generated file names the copyright holder without a year range. Protects the weekly generation from rewriting " +
"every file when the year changes.")]
public void GeneratedHeadersHaveNoYear()
{
foreach (RenderedType type in FixtureOutput.Result.Types)
{
string copyright = type.Text.Split('\n')[1];
Assert.That(copyright, Is.EqualTo("# Copyright (c) the respective contributors. All rights reserved."), type.Ref.RelativePath);
}
}
}
}
Loading
Loading