Network Extension: Orchestrate external Network devices (#13032)

This PR introduces and wires a new extension model for external network orchestration in CloudStack, centered on a new extension type: NetworkOrchestrator.

It extends the extension lifecycle from cluster-only registration to physical-network registration, adds API support for updating registered extension metadata, and enables automatic offering creation (network and VPC) based on provider-declared supported services and capabilities.

It also adds smoke coverage (KVM-only) using a Linux network namespace based implementation.

Doc PR: apache/cloudstack-documentation#649

What's new

1) New extension type: NetworkOrchestrator
Adds support for creating extensions of type NetworkOrchestrator.
Intended to back CloudStack network/VPC operations via an external orchestrator/provider.

2) Register extension with PhysicalNetwork (in addition to Cluster)
Extensions can now be registered against a PhysicalNetwork resource.
This enables network service provider behavior at physical-network scope, not only cluster scope.

3) Physical network registration details support
Registered extension details for PhysicalNetwork are handled similarly to cluster registration details.
Supports storing/updating external access metadata (credentials/endpoints/config details).

4) New API: update registered extension
Adds API support to update extension registration metadata after registration.
Useful for rotating credentials, updating endpoints, and changing external connection properties without re-registering.

5) Offering automation from external provider capabilities
Network/VPC offerings can be created with the external network provider using:
provider supportedservices
per-service service capabilities
This allows CloudStack offerings to align with what the external provider actually supports.

6) Network support via generated offerings
Using offerings backed by the external provider, networks can be created and operated with supported services/capabilities.

Supported operations include (based on provider capabilities):

Source NAT
Static NAT
Port Forwarding
Firewall
Load Balancing
DHCP
DNS
UserData

7) VPC support via generated offerings
Using VPC offerings backed by the external provider, VPCs and tiers can be created and operated with supported services/capabilities.

Supported operations include (based on provider capabilities):

VPC tier creation/implementation
Source NAT in VPC context
Static NAT / Port Forwarding / LB on VPC tiers
Network ACL association and ACL rule apply paths
Related lifecycle/restart/reapply operations

8) Linux network namespace based external implementation
Adds/uses a network extension implementation based on Linux network namespaces.
Reference implementation:
https://github.com/apache/cloudstack-extensions/tree/network-namespace/Network-Namespace

9) Smoke test coverage (KVM-only)
Adds smoke tests using the namespace-based extension implementation.
Scope includes provider lifecycle, offering creation, network/VPC flows, and key network services.
Applicable hypervisor for this smoke suite: KVM.

Author: weizhouapache

api/src/main/java/com/cloud/event/EventTypes.java

@@ -875,6 +875,7 @@ public class EventTypes {
     public static final String EVENT_EXTENSION_DELETE = "EXTENSION.DELETE";
     public static final String EVENT_EXTENSION_RESOURCE_REGISTER = "EXTENSION.RESOURCE.REGISTER";
     public static final String EVENT_EXTENSION_RESOURCE_UNREGISTER = "EXTENSION.RESOURCE.UNREGISTER";
+    public static final String EVENT_EXTENSION_RESOURCE_UPDATE = "EXTENSION.RESOURCE.UPDATE";
     public static final String EVENT_EXTENSION_CUSTOM_ACTION_ADD = "EXTENSION.CUSTOM.ACTION.ADD";
     public static final String EVENT_EXTENSION_CUSTOM_ACTION_UPDATE = "EXTENSION.CUSTOM.ACTION.UPDATE";
     public static final String EVENT_EXTENSION_CUSTOM_ACTION_DELETE = "EXTENSION.CUSTOM.ACTION.DELETE";

api/src/main/java/com/cloud/network/Network.java

@@ -116,6 +116,7 @@ class Service {
         public static final Service NetworkACL = new Service("NetworkACL", Capability.SupportedProtocols);
         public static final Service Connectivity = new Service("Connectivity", Capability.DistributedRouter, Capability.RegionLevelVpc, Capability.StretchedL2Subnet,
                 Capability.NoVlan, Capability.PublicAccess);
+        public static final Service CustomAction = new Service("CustomAction");
 
         private final String name;
         private final Capability[] caps;
@@ -207,6 +208,7 @@ public static class Provider {
 
         public static final Provider Nsx = new Provider("Nsx", false);
         public static final Provider Netris = new Provider("Netris", false);
+        public static final Provider NetworkExtension = new Provider("NetworkExtension", false, true);
 
         private final String name;
         private final boolean isExternal;
@@ -250,11 +252,47 @@ public static Provider getProvider(String providerName) {
             return null;
         }
 
+        /** Private constructor for transient (non-registered) providers. */
+        private Provider(String name) {
+            this.name = name;
+            this.isExternal = false;
+            this.needCleanupOnShutdown = true;
+            // intentionally NOT added to supportedProviders
+        }
+
+        /**
+         * Creates a transient (non-registered) {@link Provider} with the given name.
+         *
+         * <p>The new instance is <em>not</em> added to {@code supportedProviders}, so it
+         * will never be returned by {@link #getProvider(String)} and will not pollute the
+         * global provider registry.  Use this for dynamic / extension-backed providers
+         * whose names are only known at runtime (e.g. NetworkOrchestrator extensions).</p>
+         *
+         * @param name the provider name (typically the extension name)
+         * @return a transient {@link Provider} instance with the given name
+         */
+        public static Provider createTransientProvider(String name) {
+            return new Provider(name);
+        }
+
         @Override public String toString() {
             return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
                     .append("name", name)
                     .toString();
         }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) return true;
+            if (!(obj instanceof Provider)) return false;
+            Provider provider = (Provider) obj;
+            return this.name.equals(provider.name);
+        }
+
+        @Override
+        public int hashCode() {
+            return name.hashCode();
+        }
     }
 
     public static class Capability {

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -187,6 +187,8 @@ public interface NetworkModel {
 
     boolean canElementEnableIndividualServices(Provider provider);
 
+    boolean canElementEnableIndividualServicesByName(String providerName);
+
     boolean areServicesSupportedInNetwork(long networkId, Service... services);
 
     boolean isNetworkSystem(Network network);
@@ -237,6 +239,18 @@ public interface NetworkModel {
 
     String getDefaultGuestTrafficLabel(long dcId, HypervisorType vmware);
 
+    /**
+     * Resolves a provider name to a {@link Provider} instance.
+     * For known static providers, delegates to {@link Provider#getProvider(String)}.
+     * For dynamically-registered NetworkOrchestrator extension providers whose names
+     * are not in the static registry, returns a transient {@link Provider} with the
+     * given name so callers can still dispatch correctly.
+     *
+     * @param providerName the provider name from {@code ntwk_service_map} or similar
+     * @return a {@link Provider} instance, or {@code null} if not resolvable
+     */
+    Provider resolveProvider(String providerName);
+
     /**
      * @param providerName
      * @return

api/src/main/java/com/cloud/network/Networks.java

@@ -81,7 +81,11 @@ public String getValueFrom(URI uri) {
                 return uri == null ? null : uri.getAuthority();
             }
         },
-        Vswitch("vs", String.class), LinkLocal(null, null), Vnet("vnet", Long.class), Storage("storage", Integer.class), Lswitch("lswitch", String.class) {
+        Vswitch("vs", String.class),
+        LinkLocal(null, null),
+        Vnet("vnet", Long.class),
+        Storage("storage", Integer.class),
+        Lswitch("lswitch", String.class) {
             @Override
             public <T> URI toUri(T value) {
                 try {
@@ -99,7 +103,8 @@ public String getValueFrom(URI uri) {
                 return uri == null ? null : uri.getSchemeSpecificPart();
             }
         },
-        Mido("mido", String.class), Pvlan("pvlan", String.class),
+        Mido("mido", String.class),
+        Pvlan("pvlan", String.class),
         Vxlan("vxlan", Long.class) {
             @Override
             public <T> URI toUri(T value) {

api/src/main/java/com/cloud/network/element/DnsServiceProvider.java

@@ -33,4 +33,6 @@ boolean configDnsSupportForSubnet(Network network, NicProfile nic, VirtualMachin
         throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException;
 
     boolean removeDnsSupportForSubnet(Network network) throws ResourceUnavailableException;
+
+    default boolean removeDnsEntry(Network network, NicProfile nic, VirtualMachineProfile vmProfile) throws ResourceUnavailableException  { return true; }
 }

api/src/main/java/com/cloud/network/element/NetworkElement.java

@@ -146,4 +146,8 @@ boolean shutdownProviderInstances(PhysicalNetworkServiceProvider provider, Reser
      * @return true/false
      */
     boolean verifyServicesCombination(Set<Service> services);
+
+    default boolean rollingRestartSupported() {
+        return true;
+    }
 }

api/src/main/java/com/cloud/vm/NicProfile.java

@@ -463,6 +463,6 @@ public String toString() {
         return String.format("NicProfile %s",
                 ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
                         this, "id", "uuid", "vmId", "deviceId",
-                        "broadcastUri", "reservationId", "iPv4Address"));
+                        "broadcastType", "broadcastUri", "reservationId", "iPv4Address"));
     }
 }

api/src/main/java/org/apache/cloudstack/extension/CustomActionResultResponse.java

@@ -62,4 +62,12 @@ public Boolean getSuccess() {
     public void setResult(Map<String, String> result) {
         this.result = result;
     }
+
+    public Map<String, String> getResult() {
+        return result;
+    }
+
+    public boolean isSuccess() {
+        return Boolean.TRUE.equals(success);
+    }
 }

api/src/main/java/org/apache/cloudstack/extension/Extension.java

@@ -24,7 +24,8 @@
 
 public interface Extension extends InternalIdentity, Identity {
     enum Type {
-        Orchestrator
+        Orchestrator,
+        NetworkOrchestrator
     }
     enum State {
         Enabled, Disabled;

api/src/main/java/org/apache/cloudstack/extension/ExtensionCustomAction.java

@@ -48,7 +48,9 @@
 
 public interface ExtensionCustomAction extends InternalIdentity, Identity {
     enum ResourceType {
-        VirtualMachine(com.cloud.vm.VirtualMachine.class);
+        VirtualMachine(com.cloud.vm.VirtualMachine.class),
+        Network(com.cloud.network.Network.class),
+        Vpc(com.cloud.network.vpc.Vpc.class);
 
         private final Class<?> clazz;