Skip to content

Commit

Permalink
upgraded neonsdk and some perf fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@marcusbooyah
marcusbooyah committed Mar 19, 2024
1 parent 8a18bb9 commit 7fc4b4a
Show file tree
Hide file tree
Showing 13 changed files with 114 additions and 61 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,3 +159,4 @@ KubernetesWithRetry.cs
/test/test-operator/CRDs
/test/test-operator/Manifests
/test/test-operator/RBAC
*.binlog
2 changes: 1 addition & 1 deletion Directory.Build.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<Product>Neon.Operator</Product>
<Authors>NEONFORGE Team</Authors>
<Company>NEONFORGE LLC</Company>
<NeonSdkPackageVersion>4.0.5</NeonSdkPackageVersion>
<NeonSdkPackageVersion>4.0.6</NeonSdkPackageVersion>
<Copyright>Copyright © 2005-2024 by NEONFORGE LLC. All rights reserved.</Copyright>
<PackageReadmeFile Condition="Exists('README.md')">README.md</PackageReadmeFile>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
Expand Down
3 changes: 3 additions & 0 deletions src/Neon.Operator.Analyzers/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ static Constants()
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------";
public const string YamlCrdHeader = "# This CustomResourceDefinition was generated by the NeonFORGE Operator SDK.";
public const string YamlCsvHeader = "# This ClusterServiceVersion was generated by the NeonFORGE Operator SDK.";
public const string YamlExtension = ".yaml";
public const string ObjectTypeString = "object";
public const string StringTypeString = "string";
public const string BooleanTypeString = "boolean";
Expand Down
140 changes: 89 additions & 51 deletions src/Neon.Operator.Analyzers/Generators/CustomResourceDefinitionGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ public class CustomResourceDefinitionGenerator : ISourceGenerator
private static readonly string[] IgnoredProperties = { "metadata", "apiversion", "kind" };

private Dictionary<string, StringBuilder> logs;

public static XmlDocumentationProvider DocumentationProvider { get; set; } = new XmlDocumentationProvider();

public void Initialize(GeneratorInitializationContext context)
{
//System.Diagnostics.Debugger.Launch();
Expand Down Expand Up @@ -155,7 +158,7 @@ public void Execute(GeneratorExecutionContext context)
var namedTypeSymbols = context.Compilation.GetNamedTypeSymbols();
var customResourceDefinitions = new Dictionary<string, V1CustomResourceDefinition>();
var operatorVersionAttribute = RoslynExtensions.GetAttribute<VersionAttribute>(metadataLoadContext, context.Compilation, attributes);
var docProvider = new XmlDocumentationProvider(context.Compilation);
DocumentationProvider.AddMetadataReferences(context.Compilation.ExternalReferences);

var operatorVersion = operatorVersionAttribute?.Version ?? "";

Expand Down Expand Up @@ -206,30 +209,30 @@ public void Execute(GeneratorExecutionContext context)
}
var additionalPrinterColumns = new List<V1CustomResourceColumnDefinition>();

var schema = new V1CustomResourceValidation(MapType(namedTypeSymbols, crSystemType, additionalPrinterColumns, string.Empty, docProvider));
var schema = new V1CustomResourceValidation(MapType(namedTypeSymbols, crSystemType, additionalPrinterColumns, string.Empty));
var pluralNameGroup = string.IsNullOrEmpty(k8sAttr.Group) ? k8sAttr.PluralName : $"{k8sAttr.PluralName}.{k8sAttr.Group}";
var implementsStatus = crSystemType.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition().Equals(typeof(IStatus<> )));

schema.OpenAPIV3Schema.Description = crTypeIdentifier.GetSummary(docProvider);
schema.OpenAPIV3Schema.Description = crTypeIdentifier.GetSummary(DocumentationProvider);

var version = new V1CustomResourceDefinitionVersion(
name: k8sAttr.ApiVersion,
served: versionAttr?.Served ?? true,
storage: versionAttr?.Storage ?? true,
schema: schema,
subresources: new V1CustomResourceSubresources()
{
Status = implementsStatus ? new object() : null,
Scale = scaleAttr != null
? new V1CustomResourceSubresourceScale()
{
LabelSelectorPath = scaleAttr.LabelSelectorPath,
SpecReplicasPath = scaleAttr.SpecReplicasPath,
StatusReplicasPath = scaleAttr.StatusReplicasPath,
}
: null
},
additionalPrinterColumns: additionalPrinterColumns);
name: k8sAttr.ApiVersion,
served: versionAttr?.Served ?? true,
storage: versionAttr?.Storage ?? true,
schema: schema,
subresources: new V1CustomResourceSubresources()
{
Status = implementsStatus ? new object() : null,
Scale = scaleAttr != null
? new V1CustomResourceSubresourceScale()
{
LabelSelectorPath = scaleAttr.LabelSelectorPath,
SpecReplicasPath = scaleAttr.SpecReplicasPath,
StatusReplicasPath = scaleAttr.StatusReplicasPath,
}
: null
},
additionalPrinterColumns: additionalPrinterColumns);

if (customResourceDefinitions.ContainsKey(pluralNameGroup))
{
Expand All @@ -245,22 +248,22 @@ public void Execute(GeneratorExecutionContext context)
else
{
var crd = new V1CustomResourceDefinition(
apiVersion: $"{V1CustomResourceDefinition.KubeGroup}/{V1CustomResourceDefinition.KubeApiVersion}",
kind: V1CustomResourceDefinition.KubeKind,
metadata: new V1ObjectMeta(name: pluralNameGroup),
spec: new V1CustomResourceDefinitionSpec(
group: k8sAttr.Group,
names: new V1CustomResourceDefinitionNames(
kind: k8sAttr.Kind,
plural: k8sAttr.PluralName,
singular: k8sAttr.Kind.ToLowerInvariant(),
shortNames: shortNames
),
scope: scope.ToMemberString(),
versions: new List<V1CustomResourceDefinitionVersion>
{
version,
}));
apiVersion: $"{V1CustomResourceDefinition.KubeGroup}/{V1CustomResourceDefinition.KubeApiVersion}",
kind: V1CustomResourceDefinition.KubeKind,
metadata: new V1ObjectMeta(name: pluralNameGroup),
spec: new V1CustomResourceDefinitionSpec(
group: k8sAttr.Group,
names: new V1CustomResourceDefinitionNames(
kind: k8sAttr.Kind,
plural: k8sAttr.PluralName,
singular: k8sAttr.Kind.ToLowerInvariant(),
shortNames: shortNames
),
scope: scope.ToMemberString(),
versions: new List<V1CustomResourceDefinitionVersion>
{
version,
}));

customResourceDefinitions.Add(pluralNameGroup, crd);
}
Expand All @@ -280,6 +283,8 @@ public void Execute(GeneratorExecutionContext context)
Directory.CreateDirectory(olmManifestDir);
}

var fileNames = new List<string>();

foreach (var crd in customResourceDefinitions)
{
try
Expand All @@ -295,16 +300,22 @@ public void Execute(GeneratorExecutionContext context)
}
else
{
var yaml = KubernetesYaml.Serialize(crd.Value);
var sb = new StringBuilder();
sb.AppendLine(Constants.YamlCrdHeader);
sb.AppendLine(KubernetesYaml.Serialize(crd.Value));
var yaml = sb.ToString();

var fileName = crd.Value.Name() + Constants.YamlExtension;
fileNames.Add(fileName);

var outputPath = Path.Combine(crdOutputDirectory, crd.Value.Name() + ".yaml");
var outputPath = Path.Combine(crdOutputDirectory, fileName);

if (!File.Exists(outputPath) || File.ReadAllText(outputPath) != yaml)
{
File.WriteAllText(outputPath, yaml);
}

outputPath = Path.Combine(olmManifestDir, crd.Value.Name() + ".yaml");
outputPath = Path.Combine(olmManifestDir, fileName);

if (!File.Exists(outputPath) || File.ReadAllText(outputPath) != yaml)
{
Expand All @@ -318,6 +329,8 @@ public void Execute(GeneratorExecutionContext context)
}
}

CleanDirectory(fileNames, crdOutputDirectory);
CleanDirectory(fileNames, olmManifestDir);
}
catch (Exception e)
{
Expand Down Expand Up @@ -358,24 +371,51 @@ public void Execute(GeneratorExecutionContext context)
}
}

private void CleanDirectory(List<string> fileNames, string path)
{
foreach (var filePath in Directory.GetFiles(path))
{
var fileName = Path.GetFileName(filePath);
if (fileNames.Contains(fileName)
|| Path.GetExtension(fileName) != Constants.YamlExtension)
{
continue;
}

var shouldDelete = false;
using (StreamReader reader = new StreamReader(filePath))
{
var line = reader.ReadLine();
if (line == Constants.YamlCrdHeader)
{
shouldDelete = true;
}
}

if (shouldDelete)
{
File.Delete(filePath);
}
}
}

private V1JSONSchemaProps MapProperty(
IEnumerable<INamedTypeSymbol> namedTypeSymbols,
System.Reflection.PropertyInfo info,
IList<V1CustomResourceColumnDefinition> additionalColumns,
string jsonPath,
XmlDocumentationProvider docProvider)
string jsonPath)
{
V1JSONSchemaProps props = null;
try
{
props = MapType(namedTypeSymbols, info.PropertyType, additionalColumns, jsonPath, docProvider);
props = MapType(namedTypeSymbols, info.PropertyType, additionalColumns, jsonPath);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}

props.Description ??= info.GetPropertySymbol().GetSummary(docProvider);
props.Description ??= info.GetPropertySymbol().GetSummary(DocumentationProvider);

// get additional printer column information
var additionalColumn = info.GetCustomAttribute<AdditionalPrinterColumnAttribute>();
Expand Down Expand Up @@ -420,8 +460,7 @@ private V1JSONSchemaProps MapType(
IEnumerable<INamedTypeSymbol> namedTypeSymbols,
Type type,
IList<V1CustomResourceColumnDefinition> additionalColumns,
string jsonPath,
XmlDocumentationProvider docProvider)
string jsonPath)
{
var typeSymbol = namedTypeSymbols.Where(nts => nts.GetFullMetadataName() == type.FullName).FirstOrDefault();

Expand All @@ -442,7 +481,7 @@ private V1JSONSchemaProps MapType(
else if (type.IsGenericType && type.GetGenericTypeDefinition().Equals(typeof(IDictionary<,>)))
{
props.Type = Constants.ObjectTypeString;
props.AdditionalProperties = MapType(namedTypeSymbols, type.GenericTypeArguments[1], additionalColumns, jsonPath, docProvider);
props.AdditionalProperties = MapType(namedTypeSymbols, type.GenericTypeArguments[1], additionalColumns, jsonPath);
}
else if (type.IsGenericType &&
type.GetGenericTypeDefinition().Equals(typeof(IEnumerable<>)) &&
Expand All @@ -451,12 +490,12 @@ private V1JSONSchemaProps MapType(
type.GenericTypeArguments.Single().GetGenericTypeDefinition().Equals(typeof(KeyValuePair<,>)))
{
props.Type = Constants.ObjectTypeString;
props.AdditionalProperties = MapType(namedTypeSymbols, type.GenericTypeArguments.Single().GenericTypeArguments[1], additionalColumns, jsonPath, docProvider);
props.AdditionalProperties = MapType(namedTypeSymbols, type.GenericTypeArguments.Single().GenericTypeArguments[1], additionalColumns, jsonPath);
}
else if (type.IsGenericType && type.IsEnumerableType(out Type typeParameter))
{
props.Type = Constants.ArrayTypeString;
props.Items = MapType(namedTypeSymbols, typeParameter, additionalColumns, jsonPath, docProvider);
props.Items = MapType(namedTypeSymbols, typeParameter, additionalColumns, jsonPath);
}

else if (typeof(IKubernetesObject).IsAssignableFrom(type) &&
Expand All @@ -473,8 +512,7 @@ private V1JSONSchemaProps MapType(
namedTypeSymbols,
type.GetElementType() ?? throw new NullReferenceException("No Array Element Type found"),
additionalColumns,
jsonPath,
docProvider);
jsonPath);
}
else if (type.Equals(typeof(IntstrIntOrString)))
{
Expand Down Expand Up @@ -542,7 +580,7 @@ private V1JSONSchemaProps MapType(
continue;
}

props.Properties.Add(GetPropertyName(prop), MapProperty(namedTypeSymbols, prop, additionalColumns, $"{jsonPath}.{GetPropertyName(prop)}", docProvider));
props.Properties.Add(GetPropertyName(prop), MapProperty(namedTypeSymbols, prop, additionalColumns, $"{jsonPath}.{GetPropertyName(prop)}"));
};

props.Required = type.GetProperties()
Expand Down
8 changes: 6 additions & 2 deletions src/Neon.Operator.Analyzers/Generators/OlmGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -664,7 +664,11 @@ public void Execute(GeneratorExecutionContext context)
}
}

var outputString = KubernetesHelper.YamlSerialize(csv);
var sb = new StringBuilder();
sb.AppendLine(Constants.YamlCsvHeader);
sb.AppendLine(KubernetesHelper.YamlSerialize(csv));

var outputString = sb.ToString();
var outputBaseDir = Path.Combine(targetDir, "OperatorLifecycleManager");
var versionDir = Path.Combine(outputBaseDir, version);
var manifestDir = Path.Combine(versionDir, "manifests");
Expand All @@ -675,7 +679,7 @@ public void Execute(GeneratorExecutionContext context)
Directory.CreateDirectory(manifestDir);
Directory.CreateDirectory(metadataDir);

var csvPath = Path.Combine(manifestDir, $"{operatorName?.Name.ToLower()}.clusterserviceversion.yaml");
var csvPath = Path.Combine(manifestDir, $"{operatorName?.Name.ToLower()}.clusterserviceversion{Constants.YamlExtension}");
File.WriteAllText(csvPath, outputString);

if (maintainers?.Any(m => m.Value.Reviewer) == true
Expand Down
6 changes: 3 additions & 3 deletions src/Neon.Operator.Analyzers/Generators/RbacRuleGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ public void Execute(GeneratorExecutionContext context)
saNameString.Append($"-{sa.Namespace()}");
}

saNameString.Append(".yaml");
saNameString.Append(Constants.YamlExtension);

File.WriteAllText(Path.Combine(rbacOutputDirectory, saNameString.ToString()), saString);
}
Expand Down Expand Up @@ -523,7 +523,7 @@ public void Execute(GeneratorExecutionContext context)
rNameString.Append($"-{r.Namespace()}");
}

rNameString.Append(".yaml");
rNameString.Append(Constants.YamlExtension);

File.WriteAllText(Path.Combine(rbacOutputDirectory, rNameString.ToString()), rString);
}
Expand All @@ -540,7 +540,7 @@ public void Execute(GeneratorExecutionContext context)
rbNameString.Append($"-{rb.Namespace()}");
}

rbNameString.Append(".yaml");
rbNameString.Append(Constants.YamlExtension);

var outputPath = Path.Combine(rbacOutputDirectory, rbNameString.ToString());

Expand Down
2 changes: 0 additions & 2 deletions src/Neon.Operator.Analyzers/RoslynExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;

Expand Down
1 change: 1 addition & 0 deletions test/Test.Analyzers/Outputs/examples.example.neonkube.io.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# This CustomResourceDefinition was generated by the NeonFORGE Operator SDK.
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
Expand Down
1 change: 1 addition & 0 deletions test/Test.Analyzers/Outputs/generics.example.neonkube.io.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# This CustomResourceDefinition was generated by the NeonFORGE Operator SDK.
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
Expand Down
3 changes: 2 additions & 1 deletion test/Test.Analyzers/Test_Crds.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ namespace TestNamespace
/// <summary>
/// The status.
/// </summary>
[KubernetesEntity(Group = ""example.neonkube.io"", Kind = KubeKind, ApiVersion = KubeApiVersion, PluralName = KubePlural)]
[KubernetesEntity(Group = ""example.neonkube.io"", Kind = ""Example"", ApiVersion = ""v1alpha1"", PluralName = ""examples"")]
public class V1Example
{{
/// <summary>
Expand All @@ -163,6 +163,7 @@ public class V1Example
.AddSource(source)
.AddAssembly(typeof(V1Condition).Assembly)
.AddOption("build_property.NeonOperatorGenerateCrds", true)
.AddOption("build_property.TargetDir", tempFile.Path)
.AddOption("build_property.NeonOperatorCrdOutputDir", tempFile.Path)
.Build();

Expand Down
5 changes: 5 additions & 0 deletions test/test-operator/Entities/V1ExampleEntity.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ public class V1ExampleSpec<T>
/// The <see cref="V1ExampleEntity.Person"/>
/// </summary>
public Person Person { get; set; }

/// <summary>
/// Resources
/// </summary>
public V1ResourceRequirements Resources { get; set; }
}

/// <summary>
Expand Down
2 changes: 1 addition & 1 deletion test/test-operator/Entities/V2ExampleEntity.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public class V2ExampleEntity : IKubernetesObject<V1ObjectMeta>, ISpec<V2ExampleE
/// <summary>
/// The plural name of the entity.
/// </summary>
public const string KubePlural = "examples";
public const string KubePlural = V1ExampleEntity.KubePlural;

/// <summary>
/// Constructor.
Expand Down
Loading

0 comments on commit 7fc4b4a

Please sign in to comment.