diff --git a/.gitignore b/.gitignore index 1cbbcbf7..5820d6be 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ state.yaml plan.json test.py /generated/ +/.apt_generated/ +/.apt_generated_tests/ diff --git a/Dockerfile b/Dockerfile index cedf3942..9be336b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,11 @@ -FROM openjdk:8-jre-slim +FROM gradle:5.6.4-jdk8 AS build +COPY --chown=gradle:gradle . /home/gradle/src +WORKDIR /home/gradle/src +RUN gradle clean build buildRelease -x test +FROM openjdk:8-jre-slim RUN apt-get update && apt-get --yes upgrade && \ apt-get install -y python3 python3-pip curl && \ rm -rf /var/lib/apt/lists/* -COPY ./build/output/kafka-gitops /usr/local/bin/kafka-gitops \ No newline at end of file +COPY --from=build /home/gradle/src/build/output/kafka-gitops /usr/local/bin/kafka-gitops \ No newline at end of file diff --git a/build.gradle b/build.gradle index aa2e54de..62dbd2de 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ plugins { id 'application' id 'idea' id 'jacoco' - id 'org.inferred.processors' version '1.2.10' + id 'org.inferred.processors' version '2.1.0' id "net.ltgt.apt" version "0.21" id 'com.github.johnrengelman.shadow' version '4.0.4' } @@ -19,25 +19,36 @@ sourceCompatibility = 1.8 repositories { mavenCentral() + maven { + url "https://packages.confluent.io/maven/" + } + maven { + url "https://jitpack.io" + } } dependencies { - compile group: 'org.apache.kafka', name: 'kafka-clients', version: '2.4.0' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1' - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.8" - compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.10.2" - compile 'info.picocli:picocli:4.1.4' - - compile 'org.slf4j:slf4j-api:1.7.30' - compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' - compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.3' - - processor 'org.inferred:freebuilder:2.5.0' - - testCompile group: 'junit', name: 'junit', version: '4.12' - testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.4.4' - testCompile group: 'org.spockframework', name: 'spock-core', version: '1.0-groovy-2.4' - testCompile group: 'cglib', name: 'cglib-nodep', version: '2.2' + compile group: 'org.apache.kafka', name: 'kafka-clients', version: '2.8.1' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.2.2' + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.2" + compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.2" + compile 'info.picocli:picocli:4.6.3' + + implementation ('io.confluent:kafka-schema-registry-client:7.3.3') + implementation ('io.confluent:kafka-json-schema-provider:7.3.3') + implementation ('io.confluent:kafka-protobuf-serializer:7.3.3') + implementation ('io.github.java-diff-utils:java-diff-utils:4.11') + + compile 'org.slf4j:slf4j-api:1.7.36' + compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.11' + compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.11' + + processor 'org.inferred:freebuilder:2.7.0' + + testCompile group: 'junit', name: 'junit', version: '4.13.2' + testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.5.16' + testCompile group: 'org.spockframework', name: 'spock-core', version: '1.3-groovy-2.5' + testCompile group: 'cglib', name: 'cglib-nodep', version: '2.2.2' testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.19.0' testCompile group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.0' } @@ -46,7 +57,7 @@ jacocoTestReport { reports { xml.enabled = true html.enabled = true - xml.destination "${buildDir}/reports/jacoco/report.xml" + xml.destination file("${buildDir}/reports/jacoco/report.xml") } afterEvaluate { @@ -77,4 +88,4 @@ task buildRelease(type: Zip, group: "build") { } buildRelease.dependsOn buildExecutableJar -buildExecutableJar.dependsOn shadowJar \ No newline at end of file +buildExecutableJar.dependsOn shadowJar diff --git a/docker/config/registry_jaas.conf b/docker/config/registry_jaas.conf new file mode 100644 index 00000000..7745b0dd --- /dev/null +++ b/docker/config/registry_jaas.conf @@ -0,0 +1,5 @@ +KafkaClient { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="test" + password="test-secret"; +}; diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2d5ae4f2..e7fbd59b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,24 +1,33 @@ -version: '2.1' +version: '3.3' services: zoo1: - image: zookeeper:3.4.9 + image: zookeeper:3.8.0 hostname: zoo1 ports: - "2181:2181" + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 environment: - ZOO_MY_ID: 1 - ZOO_PORT: 2181 - ZOO_SERVERS: server.1=zoo1:2888:3888 - volumes: - - ./data/zoo1/data:/data - - ./data/zoo1/datalog:/datalog + - ZOOKEEPER_SERVER_ID=1 + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + - ZOOKEEPER_INIT_LIMIT=5 + - ZOOKEEPER_SYNC_LIMIT=2 + - ZOOKEEPER_SERVERS=zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:5.5.3 + image: confluentinc/cp-kafka:7.3.3 + user: "0:0" hostname: kafka1 ports: - "9092:9092" + restart: on-failure:3 + healthcheck: + test: ps augwwx | egrep [S]upportedKafka environment: KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT @@ -27,24 +36,28 @@ services: KAFKA_BROKER_ID: 1 KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_DELETE_TOPIC_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN ZOOKEEPER_SASL_ENABLED: "false" - KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" + KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer" KAFKA_SUPER_USERS: "User:test;User:kafka" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" volumes: - - ./data/kafka1/data:/var/lib/kafka/data - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf depends_on: - zoo1 kafka2: - image: confluentinc/cp-kafka:5.5.3 + image: confluentinc/cp-kafka:7.3.3 + user: "0:0" hostname: kafka2 ports: - "9093:9093" + restart: on-failure:3 + healthcheck: + test: ps augwwx | egrep [S]upportedKafka environment: KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT @@ -53,24 +66,28 @@ services: KAFKA_BROKER_ID: 2 KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_DELETE_TOPIC_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN ZOOKEEPER_SASL_ENABLED: "false" - KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" + KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer" KAFKA_SUPER_USERS: "User:test;User:kafka" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" volumes: - - ./data/kafka2/data:/var/lib/kafka/data - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf depends_on: - zoo1 kafka3: - image: confluentinc/cp-kafka:5.5.3 + image: confluentinc/cp-kafka:7.3.3 + user: "0:0" hostname: kafka3 ports: - "9094:9094" + restart: on-failure:3 + healthcheck: + test: ps augwwx | egrep [S]upportedKafka environment: KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT @@ -79,16 +96,38 @@ services: KAFKA_BROKER_ID: 3 KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_DELETE_TOPIC_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN ZOOKEEPER_SASL_ENABLED: "false" - KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" + KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer" KAFKA_SUPER_USERS: "User:test;User:kafka" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" volumes: - - ./data/kafka3/data:/var/lib/kafka/data - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf depends_on: - zoo1 + schema-registry: + image: confluentinc/cp-schema-registry:7.3.3 + hostname: schema-registry + ports: + - "8082:8082" + restart: on-failure:5 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka1:19092,kafka2:19092,kafka3:19092" + SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT + SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN + SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8082" + SCHEMA_REGISTRY_GROUP_ID: "schema-registry-test" + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/registry_jaas.conf" + SCHEMA_REGISTRY_OPTS: "-Djava.security.auth.login.config=/etc/kafka/registry_jaas.conf" + volumes: + - ./config/registry_jaas.conf:/etc/kafka/registry_jaas.conf + depends_on: + - kafka1 + - kafka2 + - kafka3 + diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 28861d27..457aad0d 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 115e6ac0..5028f28f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 68e914be..af6708ff 100755 --- a/gradlew +++ b/gradlew @@ -28,7 +28,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" @@ -66,7 +66,7 @@ esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar -# Determine the Java cli to use to start the JVM. +# Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables @@ -82,7 +82,7 @@ location of your Java installation." fi else JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' cli could be found in your PATH. + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." @@ -161,7 +161,7 @@ save () { } APP_ARGS=$(save "$@") -# Collect all arguments for the java cli, following the shell quoting and substitution rules +# Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong diff --git a/gradlew.bat b/gradlew.bat index e95643d6..0f8d5937 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/src/main/java/com/devshawn/kafka/gitops/MainCommand.java b/src/main/java/com/devshawn/kafka/gitops/MainCommand.java index 8aa91dc3..b653f01b 100644 --- a/src/main/java/com/devshawn/kafka/gitops/MainCommand.java +++ b/src/main/java/com/devshawn/kafka/gitops/MainCommand.java @@ -84,4 +84,4 @@ public static void main(String[] args) { int exitCode = new CommandLine(new MainCommand()).execute(args); System.exit(exitCode); } -} \ No newline at end of file +} diff --git a/src/main/java/com/devshawn/kafka/gitops/StateManager.java b/src/main/java/com/devshawn/kafka/gitops/StateManager.java index c91c97b8..8b51a5a2 100644 --- a/src/main/java/com/devshawn/kafka/gitops/StateManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/StateManager.java @@ -1,10 +1,16 @@ package com.devshawn.kafka.gitops; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import com.devshawn.kafka.gitops.config.KafkaGitopsConfig; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.LoggerFactory; import com.devshawn.kafka.gitops.config.KafkaGitopsConfigLoader; import com.devshawn.kafka.gitops.config.ManagerConfig; +import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; import com.devshawn.kafka.gitops.domain.confluent.ServiceAccount; import com.devshawn.kafka.gitops.domain.options.GetAclOptions; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; @@ -12,8 +18,10 @@ import com.devshawn.kafka.gitops.domain.state.CustomAclDetails; import com.devshawn.kafka.gitops.domain.state.DesiredState; import com.devshawn.kafka.gitops.domain.state.DesiredStateFile; +import com.devshawn.kafka.gitops.domain.state.SchemaDetails; import com.devshawn.kafka.gitops.domain.state.TopicDetails; import com.devshawn.kafka.gitops.domain.state.service.KafkaStreamsService; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import com.devshawn.kafka.gitops.exception.ConfluentCloudException; import com.devshawn.kafka.gitops.exception.InvalidAclDefinitionException; import com.devshawn.kafka.gitops.exception.MissingConfigurationException; @@ -25,30 +33,26 @@ import com.devshawn.kafka.gitops.service.KafkaService; import com.devshawn.kafka.gitops.service.ParserService; import com.devshawn.kafka.gitops.service.RoleService; +import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.LogUtil; import com.devshawn.kafka.gitops.util.StateUtil; import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.util.DefaultIndenter; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; public class StateManager { - private static org.slf4j.Logger log = LoggerFactory.getLogger(StateManager.class); - private final ManagerConfig managerConfig; private final ObjectMapper objectMapper; private final ParserService parserService; private final KafkaService kafkaService; + private final SchemaRegistryService schemaRegistryService; private final RoleService roleService; private final ConfluentCloudService confluentCloudService; @@ -61,19 +65,22 @@ public StateManager(ManagerConfig managerConfig, ParserService parserService) { initializeLogger(managerConfig.isVerboseRequested()); this.managerConfig = managerConfig; this.objectMapper = initializeObjectMapper(); - KafkaGitopsConfig config = KafkaGitopsConfigLoader.load(); - this.kafkaService = new KafkaService(config); + this.kafkaService = new KafkaService(KafkaGitopsConfigLoader.load()); + this.schemaRegistryService = new SchemaRegistryService(SchemaRegistryConfigLoader.load()); this.parserService = parserService; this.roleService = new RoleService(); this.confluentCloudService = new ConfluentCloudService(objectMapper); - this.planManager = new PlanManager(managerConfig, kafkaService, objectMapper); - this.applyManager = new ApplyManager(managerConfig, kafkaService); + this.planManager = new PlanManager(managerConfig, kafkaService, schemaRegistryService, objectMapper); + this.applyManager = new ApplyManager(managerConfig, kafkaService, schemaRegistryService); } public DesiredStateFile getAndValidateStateFile() { DesiredStateFile desiredStateFile = parserService.parseStateFile(); validateTopics(desiredStateFile); validateCustomAcls(desiredStateFile); + if (schemaRegistryService.isEnabled()) { + validateSchemas(desiredStateFile); + } this.describeAclEnabled = StateUtil.isDescribeTopicAclEnabled(desiredStateFile); return desiredStateFile; } @@ -92,6 +99,9 @@ private DesiredPlan generatePlan() { planManager.planAcls(desiredState, desiredPlan); } planManager.planTopics(desiredState, desiredPlan); + if (schemaRegistryService.isEnabled()) { + planManager.planSchemas(desiredState, desiredPlan); + } return desiredPlan.build(); } @@ -107,6 +117,9 @@ public DesiredPlan apply() { if (!managerConfig.isSkipAclsDisabled()) { applyManager.applyAcls(desiredPlan); } + if (schemaRegistryService.isEnabled()) { + applyManager.applySchemas(desiredPlan); + } return desiredPlan; } @@ -143,11 +156,17 @@ private void createServiceAccount(String name, List serviceAccou private DesiredState getDesiredState() { DesiredStateFile desiredStateFile = getAndValidateStateFile(); + + // Topics DesiredState.Builder desiredState = new DesiredState.Builder() .addAllPrefixedTopicsToIgnore(getPrefixedTopicsToIgnore(desiredStateFile)); generateTopicsState(desiredState, desiredStateFile); + // Schemas + desiredState.addAllPrefixedSchemasToIgnore(getPrefixedSchemasToIgnore(desiredStateFile)); + generateSchemasState(desiredState, desiredStateFile); + if (isConfluentCloudEnabled(desiredStateFile)) { generateConfluentCloudServiceAcls(desiredState, desiredStateFile); generateConfluentCloudUserAcls(desiredState, desiredStateFile); @@ -160,7 +179,7 @@ private DesiredState getDesiredState() { } private void generateTopicsState(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) { - Optional defaultReplication = StateUtil.fetchReplication(desiredStateFile); + Optional defaultReplication = StateUtil.fetchDefaultTopicsReplication(desiredStateFile); if (defaultReplication.isPresent()) { desiredStateFile.getTopics().forEach((name, details) -> { Integer replication = details.getReplication().isPresent() ? details.getReplication().get() : defaultReplication.get(); @@ -171,6 +190,18 @@ private void generateTopicsState(DesiredState.Builder desiredState, DesiredState } } + private void generateSchemasState(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) { + Optional defaultSchemaCompatibility = StateUtil.fetchDefaultSchemasCompatibility(desiredStateFile); + if (defaultSchemaCompatibility.isPresent()) { + desiredStateFile.getSchemas().forEach((s, details) -> { + SchemaCompatibility compatibility = details.getCompatibility().isPresent() ? details.getCompatibility().get() : defaultSchemaCompatibility.get(); + desiredState.putSchemas(s, new SchemaDetails.Builder().mergeFrom(details).setCompatibility(compatibility).build()); + }); + } else { + desiredState.putAllSchemas(desiredStateFile.getSchemas()); + } + } + private void generateConfluentCloudServiceAcls(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) { List serviceAccounts = confluentCloudService.getServiceAccounts(); desiredStateFile.getServices().forEach((name, service) -> { @@ -284,6 +315,16 @@ private List getPrefixedTopicsToIgnore(DesiredStateFile desiredStateFile return topics; } + private List getPrefixedSchemasToIgnore(DesiredStateFile desiredStateFile) { + List schemas = new ArrayList<>(); + try { + schemas.addAll(desiredStateFile.getSettings().get().getSchemas().get().getBlacklist().get().getPrefixed()); + } catch (NoSuchElementException ex) { + // Do nothing, no blacklist exists + } + return schemas; + } + private GetAclOptions buildGetAclOptions(String serviceName) { return new GetAclOptions.Builder().setServiceName(serviceName).setDescribeAclEnabled(describeAclEnabled).build(); } @@ -309,7 +350,7 @@ private void validateCustomAcls(DesiredStateFile desiredStateFile) { } private void validateTopics(DesiredStateFile desiredStateFile) { - Optional defaultReplication = StateUtil.fetchReplication(desiredStateFile); + Optional defaultReplication = StateUtil.fetchDefaultTopicsReplication(desiredStateFile); if (!defaultReplication.isPresent()) { desiredStateFile.getTopics().forEach((name, details) -> { if (!details.getReplication().isPresent()) { @@ -323,6 +364,16 @@ private void validateTopics(DesiredStateFile desiredStateFile) { } } + private void validateSchemas(DesiredStateFile desiredStateFile) { + Optional defaultSchemaCompatibility = StateUtil.fetchDefaultSchemasCompatibility(desiredStateFile); + desiredStateFile.getSchemas().forEach((subject, details) -> { + if (!defaultSchemaCompatibility.isPresent() && !details.getCompatibility().isPresent()) { + throw new ValidationException(String.format("Not set: [compatibility] in state file definition: schema -> %s", subject)); + } + //Schema parsing is deferred in the PlanManager in order not to do it more than one time. + }); + } + private boolean isConfluentCloudEnabled(DesiredStateFile desiredStateFile) { if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getCcloud().isPresent()) { return desiredStateFile.getSettings().get().getCcloud().get().isEnabled(); @@ -331,11 +382,17 @@ private boolean isConfluentCloudEnabled(DesiredStateFile desiredStateFile) { } private ObjectMapper initializeObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - objectMapper.registerModule(new Jdk8Module()); - return objectMapper; + ObjectMapper gitopsObjectMapper = new ObjectMapper(); + gitopsObjectMapper.enable(SerializationFeature.INDENT_OUTPUT); + gitopsObjectMapper.enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES); + gitopsObjectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + gitopsObjectMapper.registerModule(new Jdk8Module()); + DefaultIndenter defaultIndenter = new DefaultIndenter(" ", DefaultIndenter.SYS_LF); + DefaultPrettyPrinter printer = new DefaultPrettyPrinter() + .withObjectIndenter(defaultIndenter) + .withArrayIndenter(defaultIndenter); + gitopsObjectMapper.setDefaultPrettyPrinter(printer); + return gitopsObjectMapper; } private void initializeLogger(boolean verbose) { diff --git a/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java b/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java index 0c94bd96..b3eeb6c1 100644 --- a/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java +++ b/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java @@ -8,6 +8,7 @@ import com.devshawn.kafka.gitops.exception.MissingConfigurationException; import com.devshawn.kafka.gitops.exception.PlanIsUpToDateException; import com.devshawn.kafka.gitops.exception.ReadPlanInputException; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.exception.ValidationException; import com.devshawn.kafka.gitops.service.ParserService; import com.devshawn.kafka.gitops.util.LogUtil; @@ -45,6 +46,10 @@ public Integer call() { LogUtil.printValidationResult(ex.getMessage(), false); } catch (KafkaExecutionException ex) { LogUtil.printKafkaExecutionError(ex, true); + } catch (SchemaRegistryExecutionException ex) { + LogUtil.printSchemaRegistryExecutionError(ex, true); + } catch (Exception ex) { + LogUtil.printGenericError(ex, true); } return 2; } diff --git a/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java b/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java index f8c4e8f4..b22e84a5 100644 --- a/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java +++ b/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java @@ -7,6 +7,7 @@ import com.devshawn.kafka.gitops.exception.KafkaExecutionException; import com.devshawn.kafka.gitops.exception.MissingConfigurationException; import com.devshawn.kafka.gitops.exception.PlanIsUpToDateException; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.exception.ValidationException; import com.devshawn.kafka.gitops.exception.WritePlanOutputException; import com.devshawn.kafka.gitops.service.ParserService; @@ -49,6 +50,10 @@ public Integer call() { LogUtil.printKafkaExecutionError(ex); } catch (WritePlanOutputException ex) { LogUtil.printPlanOutputError(ex); + } catch (SchemaRegistryExecutionException ex) { + LogUtil.printSchemaRegistryExecutionError(ex); + } catch (Exception ex) { + LogUtil.printGenericError(ex, false); } return 2; } diff --git a/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfig.java b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfig.java new file mode 100644 index 00000000..82e1db6c --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfig.java @@ -0,0 +1,18 @@ +package com.devshawn.kafka.gitops.config; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.Map; + +@FreeBuilder +@JsonDeserialize(builder = SchemaRegistryConfig.Builder.class) +public interface SchemaRegistryConfig { + + Map getConfig(); + + class Builder extends SchemaRegistryConfig_Builder { + + } + +} diff --git a/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java new file mode 100644 index 00000000..11130e65 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java @@ -0,0 +1,62 @@ +package com.devshawn.kafka.gitops.config; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.LoggerFactory; + +public class SchemaRegistryConfigLoader { + + private static org.slf4j.Logger log = LoggerFactory.getLogger(SchemaRegistryConfigLoader.class); + + public static final String SCHEMA_REGISTRY_URL_KEY = "SCHEMA_REGISTRY_URL"; + public static final String SCHEMA_DIRECTORY_KEY = "SCHEMA_DIRECTORY"; + public static final AtomicReference instance = new AtomicReference<>(); + + private SchemaRegistryConfigLoader() { + } + + public static SchemaRegistryConfig load() { + return instance.updateAndGet(v -> { + if (v != null) { + return v; + } + SchemaRegistryConfig.Builder builder = new SchemaRegistryConfig.Builder(); + setConfig(builder); + return builder.build(); + }); + } + + private static void setConfig(SchemaRegistryConfig.Builder builder) { + Map config = new HashMap<>(); + + Map environment = System.getenv(); + + environment.forEach((key, value) -> { + if (key.equals(SCHEMA_REGISTRY_URL_KEY)) { + config.put(SCHEMA_REGISTRY_URL_KEY, value); + } else if (key.equals(SCHEMA_DIRECTORY_KEY)) { + config.put(SCHEMA_DIRECTORY_KEY, value); + } else if (key.startsWith("SCHEMA_REGISTRY_")) { + String newKey = key.substring("SCHEMA_REGISTRY_".length()).replace("_", ".").toLowerCase(); + config.put(newKey, value); + } + }); + + handleDefaultConfig(config); + + log.info("Schema Registry Config: {}", config); + + builder.putAllConfig(config); + } + + private static void handleDefaultConfig(Map config) { + final String CURRENT_WORKING_DIR = System.getProperty("user.dir"); + if (!config.containsKey(SCHEMA_DIRECTORY_KEY)) { + log.info("{} not set. Defaulting to current working directory: {}", SCHEMA_DIRECTORY_KEY, + CURRENT_WORKING_DIR); + config.put(SCHEMA_DIRECTORY_KEY, CURRENT_WORKING_DIR); + } + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java b/src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java index 156c6284..7eca2561 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java @@ -12,12 +12,15 @@ public interface DesiredPlan { List getTopicPlans(); + List getSchemaPlans(); + List getAclPlans(); default DesiredPlan toChangesOnlyPlan() { DesiredPlan.Builder builder = new DesiredPlan.Builder(); getTopicPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).map(TopicPlan::toChangesOnlyPlan).forEach(builder::addTopicPlans); getAclPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).forEach(builder::addAclPlans); + getSchemaPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).forEach(builder::addSchemaPlans); return builder.build(); } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java b/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java new file mode 100644 index 00000000..58fbce3c --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java @@ -0,0 +1,26 @@ +package com.devshawn.kafka.gitops.domain.plan; + +import com.devshawn.kafka.gitops.domain.state.SchemaDetails; +import com.devshawn.kafka.gitops.enums.PlanAction; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.Optional; + +@FreeBuilder +@JsonDeserialize(builder = SchemaPlan.Builder.class) +public interface SchemaPlan { + + String getName(); + + PlanAction getAction(); + + Optional getSchemaDetails(); + + @JsonIgnore + Optional getDiff(); + + class Builder extends SchemaPlan_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java index d46d6d78..5e9521c9 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java @@ -14,8 +14,12 @@ public interface DesiredState { Map getAcls(); + Map getSchemas(); + List getPrefixedTopicsToIgnore(); + List getPrefixedSchemasToIgnore(); + class Builder extends DesiredState_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java index a6862fb9..35689da5 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java @@ -18,6 +18,8 @@ public interface DesiredStateFile { Map getTopics(); + Map getSchemas(); + Map getUsers(); Map> getCustomServiceAcls(); diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java new file mode 100644 index 00000000..d201d00f --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java @@ -0,0 +1,25 @@ +package com.devshawn.kafka.gitops.domain.state; + +import org.inferred.freebuilder.FreeBuilder; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference; + +@FreeBuilder +@JsonDeserialize(builder = ReferenceDetails.Builder.class) +public interface ReferenceDetails { + + String getName(); + + String getSubject(); + + Integer getVersion(); + + public default SchemaReference toSchemaReference() { + return new SchemaReference(getName(), getSubject(), getVersion()); + } + + class Builder extends ReferenceDetails_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java new file mode 100644 index 00000000..f315ab84 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java @@ -0,0 +1,75 @@ +package com.devshawn.kafka.gitops.domain.state; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.inferred.freebuilder.FreeBuilder; + +import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; +import com.devshawn.kafka.gitops.enums.SchemaType; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; +import com.devshawn.kafka.gitops.exception.ValidationException; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@FreeBuilder +@JsonDeserialize(builder = SchemaDetails.Builder.class) +public interface SchemaDetails { + + SchemaType getType(); + + String getSchema(); + + Optional getFile(); + + Optional getCompatibility(); + + List getReferences(); + + public class Builder extends SchemaDetails_Builder { + @Override + public SchemaDetails build() { + String schemaRaw; + if(super.getFile().isPresent()) { + boolean schema = true; + try { + super.getSchema(); + }catch (IllegalStateException e) { + schema = false; + } + if ( schema ) { + throw new IllegalStateException("schema and file fields cannot be both set at the same time"); + } + schemaRaw = loadSchemaFromDisk(super.getFile().get()); + + super.setFile(Optional.empty()); + } else { + try { + schemaRaw = super.getSchema(); + }catch (IllegalStateException e) { + throw new IllegalStateException("schema or file field must be provided"); + } + } + super.setSchema(schemaRaw); + return super.build(); + } + + private String loadSchemaFromDisk(String fileName) { + Map config = SchemaRegistryConfigLoader.load().getConfig(); + final String SCHEMA_DIRECTORY = config.get(SchemaRegistryConfigLoader.SCHEMA_DIRECTORY_KEY).toString(); + if (!Files.exists(Paths.get(SCHEMA_DIRECTORY + "/" + fileName))) { + throw new ValidationException(String.format("Schema file %s not found in schema directory at %s", getFile(), config.get("SCHEMA_DIRECTORY"))); + } + try { + return new String(Files.readAllBytes(Paths.get(SCHEMA_DIRECTORY + "/" + fileName)), StandardCharsets.UTF_8); + } catch (IOException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to load a schema from schema directory", ex.getMessage()); + } + } + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java index 0753769a..b94657aa 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java @@ -17,6 +17,8 @@ public interface Settings { Optional getFiles(); + Optional getSchemas(); + class Builder extends Settings_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsDirectory.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsDirectory.java new file mode 100644 index 00000000..d065aeec --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsDirectory.java @@ -0,0 +1,17 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.nio.file.Path; +import java.util.Optional; + +@FreeBuilder +@JsonDeserialize(builder = SettingsDirectory.Builder.class) +public interface SettingsDirectory { + + Optional getPath(); + + class Builder extends SettingsDirectory_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java index 49494924..ee314fb1 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java @@ -14,6 +14,8 @@ public interface SettingsFiles { Optional getTopics(); Optional getUsers(); + + Optional getSchemas(); class Builder extends SettingsFiles_Builder { } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsRegistry.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsRegistry.java new file mode 100644 index 00000000..404379e8 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsRegistry.java @@ -0,0 +1,17 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.net.URL; +import java.util.Optional; + +@FreeBuilder +@JsonDeserialize(builder = SettingsRegistry.Builder.class) +public interface SettingsRegistry { + + Optional getUrl(); + + class Builder extends SettingsRegistry_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemas.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemas.java new file mode 100644 index 00000000..56b3ce6a --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemas.java @@ -0,0 +1,22 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.Optional; + +@FreeBuilder +@JsonDeserialize(builder = SettingsSchemas.Builder.class) +public interface SettingsSchemas { + + Optional getRegistry(); + + Optional getDirectory(); + + Optional getBlacklist(); + + Optional getDefaults(); + + class Builder extends SettingsSchemas_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasBlacklist.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasBlacklist.java new file mode 100644 index 00000000..b06ba546 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasBlacklist.java @@ -0,0 +1,16 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.List; + +@FreeBuilder +@JsonDeserialize(builder = SettingsSchemasBlacklist.Builder.class) +public interface SettingsSchemasBlacklist { + + List getPrefixed(); + + class Builder extends SettingsSchemasBlacklist_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasDefaults.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasDefaults.java new file mode 100644 index 00000000..db60b91c --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasDefaults.java @@ -0,0 +1,16 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import java.util.Optional; +import org.inferred.freebuilder.FreeBuilder; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@FreeBuilder +@JsonDeserialize(builder = SettingsSchemasDefaults.Builder.class) +public interface SettingsSchemasDefaults { + + Optional getCompatibility(); + + class Builder extends SettingsSchemasDefaults_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/enums/SchemaCompatibility.java b/src/main/java/com/devshawn/kafka/gitops/enums/SchemaCompatibility.java new file mode 100644 index 00000000..34fae822 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/enums/SchemaCompatibility.java @@ -0,0 +1,11 @@ +package com.devshawn.kafka.gitops.enums; + +public enum SchemaCompatibility { + NONE, + BACKWARD, + FORWARD, + FULL, + BACKWARD_TRANSITIVE, + FORWARD_TRANSITIVE, + FULL_TRANSITIVE; +} diff --git a/src/main/java/com/devshawn/kafka/gitops/enums/SchemaType.java b/src/main/java/com/devshawn/kafka/gitops/enums/SchemaType.java new file mode 100644 index 00000000..a045abd0 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/enums/SchemaType.java @@ -0,0 +1,5 @@ +package com.devshawn.kafka.gitops.enums; + +public enum SchemaType { + AVRO, JSON, PROTOBUF; +} diff --git a/src/main/java/com/devshawn/kafka/gitops/exception/CompareSchemasException.java b/src/main/java/com/devshawn/kafka/gitops/exception/CompareSchemasException.java new file mode 100644 index 00000000..c0766d6d --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/exception/CompareSchemasException.java @@ -0,0 +1,8 @@ +package com.devshawn.kafka.gitops.exception; + +public class CompareSchemasException extends RuntimeException { + + public CompareSchemasException(String exMessage) { + super(String.format("Error comparing schemas: %s", exMessage)); + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/exception/MissingMultipleConfigurationException.java b/src/main/java/com/devshawn/kafka/gitops/exception/MissingMultipleConfigurationException.java new file mode 100644 index 00000000..9b086cc3 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/exception/MissingMultipleConfigurationException.java @@ -0,0 +1,13 @@ +package com.devshawn.kafka.gitops.exception; + +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + +public class MissingMultipleConfigurationException extends RuntimeException { + + public MissingMultipleConfigurationException(String... environmentVariableNames) { + super(String.format("Missing required configuration(s): %s", + Arrays.stream(environmentVariableNames).map(Objects::toString).collect(Collectors.joining(", ")))); + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/exception/SchemaRegistryExecutionException.java b/src/main/java/com/devshawn/kafka/gitops/exception/SchemaRegistryExecutionException.java new file mode 100644 index 00000000..cba74ca4 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/exception/SchemaRegistryExecutionException.java @@ -0,0 +1,15 @@ +package com.devshawn.kafka.gitops.exception; + +public class SchemaRegistryExecutionException extends RuntimeException { + + private final String exceptionMessage; + + public SchemaRegistryExecutionException(String message, String exceptionMessage) { + super(message); + this.exceptionMessage = exceptionMessage; + } + + public String getExceptionMessage() { + return exceptionMessage; + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java index c9704ded..044cb2d4 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java @@ -1,28 +1,40 @@ package com.devshawn.kafka.gitops.manager; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.clients.admin.AlterConfigOp; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.config.ConfigResource; + import com.devshawn.kafka.gitops.config.ManagerConfig; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; +import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; import com.devshawn.kafka.gitops.domain.plan.TopicConfigPlan; import com.devshawn.kafka.gitops.domain.plan.TopicDetailsPlan; import com.devshawn.kafka.gitops.domain.plan.TopicPlan; import com.devshawn.kafka.gitops.enums.PlanAction; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.service.KafkaService; +import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.LogUtil; -import org.apache.kafka.clients.admin.AlterConfigOp; -import org.apache.kafka.clients.admin.ConfigEntry; -import org.apache.kafka.common.Node; -import org.apache.kafka.common.config.ConfigResource; - -import java.util.*; public class ApplyManager { private final ManagerConfig managerConfig; private final KafkaService kafkaService; + private final SchemaRegistryService schemaRegistryService; - public ApplyManager(ManagerConfig managerConfig, KafkaService kafkaService) { + public ApplyManager(ManagerConfig managerConfig, KafkaService kafkaService, SchemaRegistryService schemaRegistryService) { this.managerConfig = managerConfig; this.kafkaService = kafkaService; + this.schemaRegistryService = schemaRegistryService; } public void applyTopics(DesiredPlan desiredPlan) { @@ -89,4 +101,51 @@ public void applyAcls(DesiredPlan desiredPlan) { } }); } + + public void applySchemas(DesiredPlan desiredPlan) { + final List referencedSubjects = new ArrayList(); + desiredPlan.getSchemaPlans().forEach(schemaPlan -> { + if (schemaPlan.getAction() == PlanAction.ADD || schemaPlan.getAction() == PlanAction.UPDATE) { + LogUtil.printSchemaPreApply(schemaPlan); + schemaRegistryService.register(schemaPlan); + LogUtil.printPostApply(); + } else if (schemaPlan.getAction() == PlanAction.REMOVE && !managerConfig.isDeleteDisabled()) { + LogUtil.printSchemaPreApply(schemaPlan); + if(! schemaRegistryService.deleteSubject(schemaPlan.getName(), true)) { + referencedSubjects.add(schemaPlan); + LogUtil.printDeferredApply(); + }else { + LogUtil.printPostApply(); + } + } + }); + if(!referencedSubjects.isEmpty()) { + cleanUpDeferredSchemas(referencedSubjects); + } + } + + private void cleanUpDeferredSchemas(List referencedSubjects) { + System.out.println("Applying deffered actions:\n"); + int maxRetry = 1; + while ( ! referencedSubjects.isEmpty() || maxRetry >= 10) { + for (Iterator iterator = referencedSubjects.iterator(); iterator.hasNext();) { + SchemaPlan referencedSubject = iterator.next(); + LogUtil.printSchemaPreApply(referencedSubject); + if(! schemaRegistryService.deleteSubject(referencedSubject.getName(), true)) { + LogUtil.printDeferredApply(); + }else { + LogUtil.printPostApply(); + iterator.remove(); + } + } + Collections.shuffle(referencedSubjects); + maxRetry++; + } + if(referencedSubjects.isEmpty()) { + System.out.println("Deferred actions successfully applied\n"); + } else { + throw new SchemaRegistryExecutionException( + "Deferred actions did not succeed...", "At least one reference cannot be removed"); + } + } } diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java index 4cd52b85..2d6db135 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java @@ -1,38 +1,44 @@ package com.devshawn.kafka.gitops.manager; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.config.ConfigResource; +import org.slf4j.LoggerFactory; import com.devshawn.kafka.gitops.config.ManagerConfig; import com.devshawn.kafka.gitops.domain.plan.AclPlan; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; import com.devshawn.kafka.gitops.domain.plan.PlanOverview; +import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; import com.devshawn.kafka.gitops.domain.plan.TopicConfigPlan; import com.devshawn.kafka.gitops.domain.plan.TopicDetailsPlan; import com.devshawn.kafka.gitops.domain.plan.TopicPlan; import com.devshawn.kafka.gitops.domain.state.AclDetails; import com.devshawn.kafka.gitops.domain.state.DesiredState; +import com.devshawn.kafka.gitops.domain.state.SchemaDetails; import com.devshawn.kafka.gitops.domain.state.TopicDetails; import com.devshawn.kafka.gitops.enums.PlanAction; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import com.devshawn.kafka.gitops.exception.PlanIsUpToDateException; import com.devshawn.kafka.gitops.exception.ReadPlanInputException; import com.devshawn.kafka.gitops.exception.ValidationException; import com.devshawn.kafka.gitops.exception.WritePlanOutputException; import com.devshawn.kafka.gitops.service.KafkaService; +import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.PlanUtil; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.kafka.clients.admin.Config; -import org.apache.kafka.clients.admin.ConfigEntry; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.common.acl.AclBinding; -import org.apache.kafka.common.config.ConfigResource; -import org.slf4j.LoggerFactory; -import java.io.FileNotFoundException; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; +import io.confluent.kafka.schemaregistry.ParsedSchema; +import io.confluent.kafka.schemaregistry.client.SchemaMetadata; public class PlanManager { @@ -40,11 +46,13 @@ public class PlanManager { private final ManagerConfig managerConfig; private final KafkaService kafkaService; + private final SchemaRegistryService schemaRegistryService; private final ObjectMapper objectMapper; - public PlanManager(ManagerConfig managerConfig, KafkaService kafkaService, ObjectMapper objectMapper) { + public PlanManager(ManagerConfig managerConfig, KafkaService kafkaService, SchemaRegistryService schemaRegistryService, ObjectMapper objectMapper) { this.managerConfig = managerConfig; this.kafkaService = kafkaService; + this.schemaRegistryService = schemaRegistryService; this.objectMapper = objectMapper; } @@ -220,6 +228,86 @@ public void planAcls(DesiredState desiredState, DesiredPlan.Builder desiredPlan) }); } + public void planSchemas(DesiredState desiredState, DesiredPlan.Builder desiredPlan) { + // TODO: Parallelize getting schema metadata? + Map currentSubjectSchemasMap = new HashMap<>(); + Map currentSubjectCompatibilityMap = new HashMap<>(); + List subjects = schemaRegistryService.getAllSubjects(); + subjects.forEach(subject -> { + SchemaMetadata schemaMetadata = schemaRegistryService.getLatestSchemaMetadata(subject); + currentSubjectSchemasMap.put(subject, schemaMetadata); + }); + SchemaCompatibility globalCompatibility = schemaRegistryService.getGlobalSchemaCompatibility(); + subjects.forEach(subject -> { + SchemaCompatibility compatibility = schemaRegistryService.getSchemaCompatibility(subject, globalCompatibility); + currentSubjectCompatibilityMap.put(subject, compatibility); + }); + + desiredState.getSchemas().forEach((subject, schemaDetails) -> { + // We now parse all the schemas + ParsedSchema parsedSchema = schemaRegistryService.parseSchema(subject, schemaDetails); + + //We store a properly formatted schema in the plan + SchemaDetails formattedSchemaDetails = SchemaDetails.Builder.from(schemaDetails).setSchema(parsedSchema.canonicalString()).build(); + SchemaPlan.Builder schemaPlan = new SchemaPlan.Builder() + .setName(subject) + .setSchemaDetails(formattedSchemaDetails); + + if (!currentSubjectSchemasMap.containsKey(subject)) { + log.info("[PLAN] Schema Subject '{}' does not exist; it will be created.", subject); + schemaPlan.setAction(PlanAction.ADD); + } else { + SchemaMetadata currentSchema = currentSubjectSchemasMap.get(subject); + if(! schemaDetails.getType().toString().equals(currentSchema.getSchemaType())) { + throw new ValidationException("Changing the schema type is not allowed " + + "(subject: " + subject + + ", current type: " + currentSchema.getSchemaType() + + ", new type:"+schemaDetails.getType()+")"); + } + SchemaCompatibility currentCompatibility = currentSubjectCompatibilityMap.get(subject); + if( schemaDetails.getCompatibility().get() != currentCompatibility ) { + // TODO: we should be able to do this + throw new ValidationException("Changing the subject compatibility is not allowed with kafka-gitops" + + "(subject: " + subject + + ", current compatibilty: " + currentCompatibility + + ", new compatibilty:" + schemaDetails.getCompatibility().get() + ")"); + } + String diff = schemaRegistryService.deepEquals(parsedSchema, currentSubjectSchemasMap.get(subject)); + if (diff.isEmpty()) { + log.info("[PLAN] Schema Subject '{}' exists and has not changed; it will not be updated.", subject); + schemaPlan.setAction(PlanAction.NO_CHANGE); + } else { + //test compatibility: + schemaRegistryService.testSchemaCompatibility(subject, parsedSchema); + log.info("[PLAN] Schema Subject '{}' exists and has changed; it will be updated. Actual diff:\n {}", subject, + diff); + schemaPlan.setDiff(diff); + schemaPlan.setAction(PlanAction.UPDATE); + } + } + + desiredPlan.addSchemaPlans(schemaPlan.build()); + }); + + currentSubjectSchemasMap.forEach((subject, schemaMetadata) -> { + boolean shouldIgnore = desiredState.getPrefixedSchemasToIgnore().stream().anyMatch(it -> subject.startsWith(it)); + if (shouldIgnore) { + log.info("[PLAN] Schema Subject {} due to prefix", subject); + return; + } + + if (!managerConfig.isDeleteDisabled() && desiredState.getSchemas().getOrDefault(subject, null) == null) { + log.info("[PLAN] Schema Subject '{}' exists and will be remove.", subject); + SchemaPlan schemaPlan = new SchemaPlan.Builder() + .setName(subject) + .setAction(PlanAction.REMOVE) + .build(); + + desiredPlan.addSchemaPlans(schemaPlan); + } + }); + } + public void validatePlanHasChanges(DesiredPlan desiredPlan, boolean deleteDisabled, boolean skipAclsDisabled) { PlanOverview planOverview = PlanUtil.getOverview(desiredPlan, deleteDisabled, skipAclsDisabled); if (planOverview.getAdd() == 0 && planOverview.getUpdate() == 0 && planOverview.getRemove() == 0) { diff --git a/src/main/java/com/devshawn/kafka/gitops/service/KafkaService.java b/src/main/java/com/devshawn/kafka/gitops/service/KafkaService.java index 54603e70..fd503738 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/KafkaService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/KafkaService.java @@ -178,7 +178,7 @@ public Map getTopicDescription(Set topics) { } private Map getTopicDescription(Set topics, AdminClient adminClient) throws InterruptedException, ExecutionException { - return adminClient.describeTopics(topics).all().get(); + return adminClient.describeTopics(topics).allTopicNames().get(); } public Map describeConfigsForTopics(List topicNames) { diff --git a/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java b/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java index c027f4af..105692c3 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java @@ -55,12 +55,16 @@ public DesiredStateFile parseStateFile() { DesiredStateFile usersFile = loadExternalFile(settingsFiles.getUsers().get(), "Users"); builder.putAllUsers(usersFile.getUsers()); } + if (settingsFiles.getSchemas().isPresent()) { + DesiredStateFile schemasFile = loadExternalFile(settingsFiles.getSchemas().get(), "Schemas"); + builder.putAllSchemas(schemasFile.getSchemas()); + } return builder.build(); } return desiredStateFile; } - public DesiredStateFile parseStateFile(File stateFile) { + private DesiredStateFile parseStateFile(File stateFile) { log.info("Parsing desired state file..."); try { @@ -81,7 +85,15 @@ public DesiredStateFile parseStateFile(File stateFile) { throw new ValidationException(String.format("Value '%s' is not a valid format for: [%s] in state file definition: %s", value, propertyName, joinedFields)); } catch (JsonMappingException ex) { List fields = getYamlFields(ex); - String message = ex.getCause() != null ? ex.getCause().getMessage().split("\n")[0] : ex.getMessage().split("\n")[0]; + String message = null; + if(ex.getCause() != null && ex.getCause().getMessage() != null) { + message = ex.getCause().getMessage().split("\n")[0]; + } + if(message == null && ex.getMessage() != null) { + message = ex.getMessage().split("\n")[0]; + } else { + message = "Unknown error"; + } String joinedFields = String.join(" -> ", fields); throw new ValidationException(String.format("%s in state file definition: %s", message, joinedFields)); } catch (FileNotFoundException ex) { diff --git a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java new file mode 100644 index 00000000..6224fd02 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java @@ -0,0 +1,275 @@ +package com.devshawn.kafka.gitops.service; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.devshawn.kafka.gitops.config.SchemaRegistryConfig; +import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; +import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; +import com.devshawn.kafka.gitops.domain.state.SchemaDetails; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; +import com.devshawn.kafka.gitops.enums.SchemaType; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; +import com.devshawn.kafka.gitops.exception.ValidationException; +import com.devshawn.kafka.gitops.util.HelperUtil; + +import io.confluent.kafka.schemaregistry.AbstractSchemaProvider; +import io.confluent.kafka.schemaregistry.ParsedSchema; +import io.confluent.kafka.schemaregistry.SchemaProvider; +import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider; +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaMetadata; +import io.confluent.kafka.schemaregistry.client.rest.RestService; +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference; +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; +import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider; +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider; + +public class SchemaRegistryService { + private static final Logger log = LoggerFactory.getLogger(SchemaRegistryService.class); + + private final boolean schemaRegistryEnabled; + private final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>(); + + //This client must be used only when the previous client does not expose the functionality. + private static final AtomicReference schemaRegistryRestService = new AtomicReference<>(); + + public SchemaRegistryService(SchemaRegistryConfig config) { + this.schemaRegistryEnabled = config.getConfig().containsKey(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY); + schemaRegistryRestService.updateAndGet(v -> { + if (isEnabled()) { + if (v != null) { + return v; + } + return new RestService( + config.getConfig().get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()); + } + return null; + }); + cachedSchemaRegistryClientRef.updateAndGet(v -> { + if (isEnabled()) { + if (v != null) { + return v; + } + return new CachedSchemaRegistryClient(schemaRegistryRestService.get(), 10, config.getConfig()); + } + return null; + }); + } + + public boolean isEnabled() { + return schemaRegistryEnabled; + } + + public List getAllSubjects() { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); + try { + return new ArrayList<>(cachedSchemaRegistryClient.getAllSubjects()); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get all schema registry subjects", ex.getMessage()); + } + } + + public boolean deleteSubject(String subject, boolean isPermanent) { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); + try { + // must always soft-delete + cachedSchemaRegistryClient.deleteSubject(subject); + if (isPermanent) { + cachedSchemaRegistryClient.deleteSubject(subject, true); + } + } catch (RestClientException ex) { + if(ex.getErrorCode() == 42206) { + log.debug("Error cleaning referenced schema ( {} )", subject); + return false; + } else { + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); + } + } catch (IOException ex) { + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); + } + return true; + } + + public int register(SchemaPlan schemaPlan) { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); + AbstractSchemaProvider schemaProvider = schemaProviderFromType(schemaPlan.getSchemaDetails().get().getType()); + ParsedSchema parsedSchema = parseSchema(schemaPlan.getName(), schemaPlan.getSchemaDetails().get(), + schemaProvider); + int id; + try { + id = cachedSchemaRegistryClient.register(schemaPlan.getName(), parsedSchema); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to register subject '" + + schemaPlan.getName() + "' in schema registry", ex.getMessage()); + } + try { + cachedSchemaRegistryClient.updateCompatibility(schemaPlan.getName(), + schemaPlan.getSchemaDetails().get().getCompatibility().get().toString()); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to update compatibility of the newly registered subject '" + + schemaPlan.getName() + "' in schema registry", + ex.getMessage()); + } + return id; + } + + public AbstractSchemaProvider schemaProviderFromType(SchemaType schemaType) { + AbstractSchemaProvider schemaProvider; + if (schemaType == SchemaType.AVRO) { + schemaProvider = new AvroSchemaProvider(); + } else if (schemaType == SchemaType.JSON) { + schemaProvider = new JsonSchemaProvider(); + } else if (schemaType == SchemaType.PROTOBUF) { + schemaProvider = new ProtobufSchemaProvider(); + } else { + throw new ValidationException("Unknown schema type: " + schemaType); + } + // we need to pass a schema registry client as a config because the underlying + // code can validate against the current state + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); + schemaProvider.configure( + Collections.singletonMap(SchemaProvider.SCHEMA_VERSION_FETCHER_CONFIG, schemaRegistryClient)); + return schemaProvider; + } + + public void testSchemaCompatibility(String subject, ParsedSchema parsedSchema) { + AbstractSchemaProvider schemaProvider = schemaProviderFromType(SchemaType.valueOf(parsedSchema.schemaType())); + testSchemaCompatibility(subject, parsedSchema, schemaProvider); + } + + public ParsedSchema parseSchema(String subject, SchemaDetails schemaDetails) { + AbstractSchemaProvider schemaProvider = schemaProviderFromType(schemaDetails.getType()); + return parseSchema(subject, schemaDetails, schemaProvider); + } + + private void testSchemaCompatibility(String subject, ParsedSchema parsedSchema, AbstractSchemaProvider schemaProvider) { + RestService restService = schemaRegistryRestService.get(); + try { + /* + * WARN: this does not work for TRANSITIVE compatibility types where all the versions must be tests + * we have to wait for v7.0.0 and uses: + * List differences = restService.testCompatibility(parsedSchema.canonicalString(), parsedSchema.schemaType(), + parsedSchema.references(), subject, null, true) + */ + List differences = restService.testCompatibility(parsedSchema.canonicalString(), parsedSchema.schemaType(), + parsedSchema.references(), subject, "latest", false); + if (differences != null && !differences.isEmpty()) { + /* + * There is a bug on the kafka version that we have which does not always return a reason... + * So doing it now and putting a reason if we have it. + */ + List differencesDetails = restService.testCompatibility(parsedSchema.canonicalString(), parsedSchema.schemaType(), + parsedSchema.references(), subject, "latest", true); + if(differencesDetails != null && !differencesDetails.isEmpty()) { + differences = differencesDetails; + } + throw new ValidationException(String.format("%s schema '%s' is incompatible with an earlier schema: %s", + schemaProvider.schemaType(), subject, differences)); + } + } catch (IOException | RestClientException ex) { + throw new ValidationException(String.format( + "Error thrown when attempting to check the compatibility of the new schema for '%s': %s", subject, + ex.getMessage())); + } + } + + private ParsedSchema parseSchema(String subject, SchemaDetails schemaDetails, AbstractSchemaProvider schemaProvider) { + Optional parsedSchema; + if (schemaDetails.getReferences().isEmpty()) { + parsedSchema = schemaProvider.parseSchema(schemaDetails.getSchema(), Collections.emptyList()); + if (!parsedSchema.isPresent()) { + throw new ValidationException(String.format("%s schema for subject '%s' could not be parsed.", + schemaProvider.schemaType(), subject)); + } + } else { + List schemaReferences = new ArrayList<>(); + schemaDetails.getReferences().forEach(referenceDetails -> { + SchemaReference schemaReference = new SchemaReference(referenceDetails.getName(), + referenceDetails.getSubject(), referenceDetails.getVersion()); + schemaReferences.add(schemaReference); + }); + + try { + parsedSchema = schemaProvider.parseSchema(schemaDetails.getSchema(), schemaReferences); + } catch (IllegalStateException ex) { + throw new ValidationException(String.format("Reference validation error: %s", ex.getMessage())); + } catch (RuntimeException ex) { + throw new ValidationException( + String.format("Error thrown when attempting to validate %s schema with reference: %s", subject, + ex.getMessage())); + } + if (!parsedSchema.isPresent()) { + throw new ValidationException(String.format("%s referenced schema could not be parsed for subject %s", + schemaProvider.schemaType(), subject)); + } + } + return parsedSchema.get(); + } + + public SchemaMetadata getLatestSchemaMetadata(String subject) { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); + try { + return cachedSchemaRegistryClient.getLatestSchemaMetadata(subject); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get schema metadata for subject '" + subject + "'", + ex.getMessage()); + } + } + + public SchemaCompatibility getGlobalSchemaCompatibility() { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); + try { + return SchemaCompatibility.valueOf(cachedSchemaRegistryClient.getCompatibility(null)); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get global schema compatibility", ex.getMessage()); + } + } + + public SchemaCompatibility getSchemaCompatibility(String subject, SchemaCompatibility globalCompatibility) { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); + try { + return SchemaCompatibility.valueOf(cachedSchemaRegistryClient.getCompatibility(subject)); + } catch (IOException ex) { + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get schema compatibility for subject '" + subject + "'", + ex.getMessage()); + } catch (RestClientException ex) { + if (ex.getErrorCode() == 40401 || ex.getErrorCode() == 40408) { + return globalCompatibility; + } + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get schema compatibility for subject '" + subject + "'", + ex.getMessage()); + } + } + + public String deepEquals(ParsedSchema newSchema, SchemaMetadata schemaMetadata) { + AbstractSchemaProvider schemaProvider = schemaProviderFromType(SchemaType.valueOf(newSchema.schemaType())); + ParsedSchema previousSchema = schemaProvider.parseSchema(schemaMetadata.getSchema(), schemaMetadata.getReferences()).get(); + + String diff = ""; + /* + * The comparison must be based on the canonical String representation otherwise + * some diff can be found even if there is no. + * The `deepEquals` function works on raw data which is not good so using the basic equals one. + */ + if (!previousSchema.equals(newSchema)) { + diff = HelperUtil.generateDiff(previousSchema, newSchema); + } + return diff; + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java index 103ecc15..b3ac2840 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java @@ -1,10 +1,20 @@ package com.devshawn.kafka.gitops.util; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import com.github.difflib.text.DiffRow; +import com.github.difflib.text.DiffRowGenerator; + +import io.confluent.kafka.schemaregistry.ParsedSchema; +import io.confluent.kafka.schemaregistry.avro.AvroSchema; +import io.confluent.kafka.schemaregistry.json.JsonSchema; +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; + public class HelperUtil { public static List uniqueCombine(List listOne, List listTwo) { @@ -12,4 +22,54 @@ public static List uniqueCombine(List listOne, List list set.addAll(listTwo); return new ArrayList<>(set); } + + public static String generateDiff(ParsedSchema parsedSchema1, ParsedSchema parsedSchema2) { + + List schemaStringList1 = lines(parsedSchema1); + List schemaStringList2 = lines(parsedSchema2); + + DiffRowGenerator generator = DiffRowGenerator.create() + .showInlineDiffs(false) + .inlineDiffByWord(false) + .build(); + + StringBuilder stringBuilder = new StringBuilder(); + List diffRows = generator.generateDiffRows(schemaStringList1, schemaStringList2); + for (Iterator iterator = diffRows.iterator(); iterator.hasNext();) { + if(stringBuilder.length() > 0) { + stringBuilder.append("\n"); + } + DiffRow diffRow = iterator.next(); + switch (diffRow.getTag()) { + case INSERT: + stringBuilder.append("+ ").append(diffRow.getNewLine()); + break; + case DELETE: + stringBuilder.append("- ").append(diffRow.getOldLine()); + break; + case CHANGE: + stringBuilder.append("- ").append(diffRow.getOldLine()).append("\n").append("+ ").append(diffRow.getNewLine()); + break; + default: + stringBuilder.append(diffRow.getOldLine()); + break; + } + } + + return stringBuilder.toString(); + } + + private static List lines(ParsedSchema parsedSchema) { + final String value; + if(parsedSchema instanceof AvroSchema) { + value = ((AvroSchema)parsedSchema).rawSchema().toString(true); + } else if(parsedSchema instanceof ProtobufSchema) { + value = ((ProtobufSchema)parsedSchema).rawSchema().toSchema(); + } else if(parsedSchema instanceof JsonSchema) { + value =((JsonSchema) parsedSchema).toJsonNode().toPrettyString(); + } else { + value = parsedSchema.canonicalString(); + } + return Arrays.asList(value.split("\n")); + } } diff --git a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java index 49919693..3c8d23ae 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java @@ -3,12 +3,14 @@ import com.devshawn.kafka.gitops.domain.plan.AclPlan; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; import com.devshawn.kafka.gitops.domain.plan.PlanOverview; +import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; import com.devshawn.kafka.gitops.domain.plan.TopicConfigPlan; import com.devshawn.kafka.gitops.domain.plan.TopicDetailsPlan; import com.devshawn.kafka.gitops.domain.plan.TopicPlan; import com.devshawn.kafka.gitops.domain.state.AclDetails; import com.devshawn.kafka.gitops.enums.PlanAction; import com.devshawn.kafka.gitops.exception.KafkaExecutionException; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.exception.WritePlanOutputException; import picocli.CommandLine; @@ -25,6 +27,9 @@ public static void printPlan(DesiredPlan desiredPlan, boolean deleteDisabled, bo printAclOverview(desiredPlan, deleteDisabled); desiredPlan.getAclPlans().forEach(LogUtil::printAclPlan); + printSchemaOverview(desiredPlan, deleteDisabled); + desiredPlan.getSchemaPlans().forEach(LogUtil::printSchemaPlan); + printOverview(desiredPlan, deleteDisabled, skipAclsDisabled); } @@ -59,6 +64,8 @@ private static void printTopicPlan(TopicPlan topicPlan) { System.out.println(red(String.format("- [TOPIC] %s", topicPlan.getName()))); System.out.println("\n"); break; + case NO_CHANGE: + break; } } @@ -112,7 +119,7 @@ private static void printTopicConfigPlan(TopicConfigPlan topicConfigPlan) { System.out.println(red(String.format("\t\t- %s (%s)", topicConfigPlan.getKey(), topicConfigPlan.getPreviousValue().get()))); break; case NO_CHANGE: - break; + break; } } @@ -142,6 +149,46 @@ private static void printAclPlan(AclPlan aclPlan) { System.out.println(red(String.format("\t - permission: %s", aclDetails.getPermission()))); System.out.println("\n"); break; + case UPDATE: + case NO_CHANGE: + break; + } + } + + private static void printSchemaPlan(SchemaPlan schemaPlan) { + switch (schemaPlan.getAction()) { + case ADD: + System.out.println(green(String.format("+ [SCHEMA] %s", schemaPlan.getName()))); + System.out.println(green(String.format("\t + type: %s", schemaPlan.getSchemaDetails().get().getType()))); + if(schemaPlan.getSchemaDetails().get().getCompatibility().isPresent()) { + System.out.println(green(String.format("\t + compatibility: %s", schemaPlan.getSchemaDetails().get().getCompatibility().get()))); + } + System.out.println(green(String.format("\t + schema:\n----------------------\n%s\n----------------------", + schemaPlan.getSchemaDetails().get().getSchema()))); + if (!schemaPlan.getSchemaDetails().get().getReferences().isEmpty()) { + schemaPlan.getSchemaDetails().get().getReferences().forEach(referenceDetail -> { + System.out.println(green("\t + references:")); + System.out.println(green(String.format("\t\t + name: %s", referenceDetail.getName()))); + System.out.println(green(String.format("\t\t + subject: %s", referenceDetail.getSubject()))); + System.out.println(green(String.format("\t\t + version: %s", referenceDetail.getVersion()))); + }); + } + System.out.println("\n"); + break; + case UPDATE: + System.out.println(yellow(String.format("~ [SCHEMA] %s", schemaPlan.getName()))); + if(schemaPlan.getDiff().isPresent()) { + System.out.println(yellow(String.format("\t ~ diff:\n----------------------\n%s\n----------------------", + schemaPlan.getDiff().get()))); + } + System.out.println("\n"); + break; + case REMOVE: + System.out.println(red(String.format("- [SCHEMA] %s", schemaPlan.getName()))); + System.out.println("\n"); + break; + case NO_CHANGE: + break; } } @@ -154,6 +201,11 @@ public static void printTopicPreApply(TopicPlan topicPlan) { printTopicPlan(topicPlan); } + public static void printSchemaPreApply(SchemaPlan schemaPlan) { + System.out.println(String.format("Applying: [%s]\n", toAction(schemaPlan.getAction()))); + printSchemaPlan(schemaPlan); + } + public static void printAclPreApply(AclPlan aclPlan) { System.out.println(String.format("Applying: [%s]\n", toAction(aclPlan.getAction()))); printAclPlan(aclPlan); @@ -163,6 +215,10 @@ public static void printPostApply() { System.out.println("Successfully applied.\n"); } + public static void printDeferredApply() { + System.out.println("Applied deferred...\n"); + } + /* * Helpers */ @@ -185,6 +241,12 @@ private static void printAclOverview(DesiredPlan desiredPlan, boolean deleteDisa toUpdate(aclPlanOverview.getUpdate()), toDelete(aclPlanOverview.getRemove()))); } + private static void printSchemaOverview(DesiredPlan desiredPlan, boolean deleteDisabled) { + PlanOverview schemaPlanOverview = PlanUtil.getSchemaPlanOverview(desiredPlan, deleteDisabled); + System.out.println(String.format("Schemas: %s, %s, %s.\n", toCreate(schemaPlanOverview.getAdd()), + toUpdate(schemaPlanOverview.getUpdate()), toDelete(schemaPlanOverview.getRemove()))); + } + private static void printLegend(PlanOverview planOverview) { System.out.println("An execution plan has been generated and is shown below."); System.out.println("Resource actions are indicated with the following symbols:"); @@ -225,7 +287,7 @@ public static void printGenericError(RuntimeException ex) { printGenericError(ex, false); } - public static void printGenericError(RuntimeException ex, boolean apply) { + public static void printGenericError(Exception ex, boolean apply) { System.out.println(String.format("[%s] %s\n", red("ERROR"), ex.getMessage())); if (apply) { printApplyErrorMessage(); @@ -247,6 +309,19 @@ public static void printKafkaExecutionError(KafkaExecutionException ex, boolean } } + public static void printSchemaRegistryExecutionError(SchemaRegistryExecutionException ex) { + printSchemaRegistryExecutionError(ex, false); + } + + public static void printSchemaRegistryExecutionError(SchemaRegistryExecutionException ex, boolean apply) { + System.out.println(String.format("[%s] %s:\n%s\n", red("ERROR"), ex.getMessage(), ex.getExceptionMessage())); + if (apply) { + printApplyErrorMessage(); + } else { + printPlanErrorMessage(); + } + } + public static void printPlanOutputError(WritePlanOutputException ex) { System.out.println(String.format("[%s] %s", red("ERROR"), ex.getMessage())); } @@ -297,7 +372,8 @@ private static String toAction(PlanAction planAction) { return yellow("UPDATE"); case REMOVE: return red("DELETE"); + default: + return null; } - return null; } } diff --git a/src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java index ff86ca95..c5af87b3 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java @@ -15,6 +15,7 @@ public static PlanOverview getOverview(DesiredPlan desiredPlan, boolean deleteDi if(!skipAclsDisabled) { desiredPlan.getAclPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); } + desiredPlan.getSchemaPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); return buildPlanOverview(map); } @@ -30,6 +31,12 @@ public static PlanOverview getAclPlanOverview(DesiredPlan desiredPlan, boolean d return buildPlanOverview(map); } + public static PlanOverview getSchemaPlanOverview(DesiredPlan desiredPlan, boolean deleteDisabled) { + EnumMap map = getPlanActionMap(); + desiredPlan.getSchemaPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); + return buildPlanOverview(map); + } + private static void addToMap(EnumMap map, PlanAction planAction, boolean deleteDisabled) { if (!(deleteDisabled && planAction == PlanAction.REMOVE)) { map.put(planAction, map.get(planAction) + 1); diff --git a/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java index 34879b9e..140e22a7 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java @@ -1,12 +1,12 @@ package com.devshawn.kafka.gitops.util; import com.devshawn.kafka.gitops.domain.state.DesiredStateFile; - +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import java.util.Optional; public class StateUtil { - public static Optional fetchReplication(DesiredStateFile desiredStateFile) { + public static Optional fetchDefaultTopicsReplication(DesiredStateFile desiredStateFile) { if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getTopics().isPresent() && desiredStateFile.getSettings().get().getTopics().get().getDefaults().isPresent()) { return desiredStateFile.getSettings().get().getTopics().get().getDefaults().get().getReplication(); @@ -20,4 +20,12 @@ public static boolean isDescribeTopicAclEnabled(DesiredStateFile desiredStateFil && desiredStateFile.getSettings().get().getServices().get().getAcls().get().getDescribeTopicEnabled().isPresent() && desiredStateFile.getSettings().get().getServices().get().getAcls().get().getDescribeTopicEnabled().get(); } + + public static Optional fetchDefaultSchemasCompatibility(DesiredStateFile desiredStateFile) { + if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getSchemas().isPresent() + && desiredStateFile.getSettings().get().getSchemas().get().getDefaults().isPresent()) { + return desiredStateFile.getSettings().get().getSchemas().get().getDefaults().get().getCompatibility(); + } + return Optional.empty(); + } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy index 130d4ecc..1aeef022 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy @@ -1,7 +1,17 @@ package com.devshawn.kafka.gitops +import java.nio.file.Path +import java.nio.file.Paths + import org.junit.Rule import org.junit.contrib.java.lang.system.EnvironmentVariables +import org.skyscreamer.jsonassert.JSONAssert + +import com.devshawn.kafka.gitops.enums.SchemaCompatibility +import com.devshawn.kafka.gitops.enums.SchemaType + +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference import picocli.CommandLine import spock.lang.Specification import spock.lang.Unroll @@ -13,16 +23,20 @@ class ApplyCommandIntegrationSpec extends Specification { EnvironmentVariables environmentVariables void setup() { - environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092") + environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092,localhost:9093,localhost:9094") environmentVariables.set("KAFKA_SASL_JAAS_USERNAME", "test") environmentVariables.set("KAFKA_SASL_JAAS_PASSWORD", "test-secret") environmentVariables.set("KAFKA_SASL_MECHANISM", "PLAIN") environmentVariables.set("KAFKA_SECURITY_PROTOCOL", "SASL_PLAINTEXT") - TestUtils.cleanUpCluster() + environmentVariables.set("SCHEMA_REGISTRY_URL", "http://localhost:8082") + Path resourceDirectory = Paths.get("src","test","resources", "plans", "schema_registry", "schemas"); + String absolutePath = resourceDirectory.toFile().getAbsolutePath(); + environmentVariables.set("SCHEMA_DIRECTORY", absolutePath) + TestUtils.cleanUpAll() } void cleanupSpec() { -// TestUtils.cleanUpCluster() +// TestUtils.cleanUpAll() } void 'test various successful applies - #planFile'() { @@ -89,7 +103,7 @@ class ApplyCommandIntegrationSpec extends Specification { void 'test various valid applies with seed - #planFile #deleteDisabled'() { setup: - TestUtils.seedCluster() + TestUtils.seedKafkaCluster() ByteArrayOutputStream out = new ByteArrayOutputStream() PrintStream oldOut = System.out System.setOut(new PrintStream(out)) @@ -170,4 +184,101 @@ class ApplyCommandIntegrationSpec extends Specification { System.setOut(oldOut) } + void 'test various successful applies schemas - #planFile'() { + setup: + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldOut = System.out + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planFile}-plan.json") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "apply", "-p", file) + + then: + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planFile}-apply-output.txt") + exitCode == 0 + + cleanup: + System.setOut(oldOut) + + where: + planFile << [ + "schema-registry-new-json", + "schema-registry-new-avro", + "schema-registry-new-proto", + "schema-registry-default", + "schema-registry-mix" + ] + } + + void 'test various valid schema registry applies with seed - #planFile #deleteDisabled'() { + setup: + TestUtils.seedSchemaRegistry() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldOut = System.out + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planFile}-plan.json") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = -1 + if (deleteDisabled) { + exitCode = cmd.execute("-f", file, "--no-delete", "apply", "-p", file) + } else { + exitCode = cmd.execute("-f", file, "apply", "-p", file) + } + + then: + if (deleteDisabled) { + assert out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planFile}-no-delete-apply-output.txt") + } else { + assert out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planFile}-apply-output.txt") + } + exitCode == 0 + + cleanup: + System.setOut(oldOut) + + where: + planFile | deleteDisabled + "seed-schema-modification" | false + "seed-schema-modification-2" | false + "seed-schema-modification-3" | false + "seed-schema-modification-4" | false + "seed-schema-modification" | true + "seed-schema-add-with-reference" | false + } + + void 'test Specific for next deferred apply with manual seed'() { + setup: + CachedSchemaRegistryClient schemaRegistryClient = TestUtils.cachedSchemaRegistryClientRef.get() + TestUtils.seedSchemaRegistry() + TestUtils.createSchema("schema-10-json", SchemaType.JSON, "{\"type\":\"object\",\"properties\":{\"test2\":{\"type\":\"string\"}}, \"additionalProperties\": false}", + schemaRegistryClient, SchemaCompatibility.BACKWARD) + List reference = new ArrayList() + reference.add(new SchemaReference("otherschema", "schema-10-json", 1)) + List subjects = schemaRegistryClient.getAllVersions("schema-10-json"); + TestUtils.createSchema("schema-11-json", SchemaType.JSON, '{"type":"object","properties":{"test2":{"\$ref":"otherschema"}}, "additionalProperties": false}', + schemaRegistryClient, SchemaCompatibility.BACKWARD, reference) + + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldOut = System.out + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/seed-schema-delete-reference-plan.json") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("apply", "-p", file) + + then: + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/seed-schema-delete-reference-apply-output.txt") + exitCode == 0 + + cleanup: + System.setOut(oldOut) + } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy index c2c0ba23..2ad14bf9 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy @@ -1,8 +1,16 @@ package com.devshawn.kafka.gitops +import java.nio.file.Path +import java.nio.file.Paths import org.junit.ClassRule import org.junit.contrib.java.lang.system.EnvironmentVariables import org.skyscreamer.jsonassert.JSONAssert + +import com.devshawn.kafka.gitops.enums.SchemaCompatibility +import com.devshawn.kafka.gitops.enums.SchemaType + +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference import picocli.CommandLine import spock.lang.Shared import spock.lang.Specification @@ -16,16 +24,20 @@ class PlanCommandIntegrationSpec extends Specification { EnvironmentVariables environmentVariables void setupSpec() { - environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092") + environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092,localhost:9093,localhost:9094") environmentVariables.set("KAFKA_SASL_JAAS_USERNAME", "test") environmentVariables.set("KAFKA_SASL_JAAS_PASSWORD", "test-secret") environmentVariables.set("KAFKA_SASL_MECHANISM", "PLAIN") environmentVariables.set("KAFKA_SECURITY_PROTOCOL", "SASL_PLAINTEXT") - TestUtils.cleanUpCluster() + environmentVariables.set("SCHEMA_REGISTRY_URL", "http://localhost:8082") + Path resourceDirectory = Paths.get("src","test","resources", "plans", "schema_registry", "schemas"); + String absolutePath = resourceDirectory.toFile().getAbsolutePath(); + environmentVariables.set("SCHEMA_DIRECTORY", absolutePath) + TestUtils.cleanUpAll() } void cleanupSpec() { -// TestUtils.cleanUpCluster() + // TestUtils.cleanUpAll() } void 'test various valid plans - #planName'() { @@ -50,24 +62,25 @@ class PlanCommandIntegrationSpec extends Specification { where: planName << [ - "simple", - "application-service", - "kafka-connect-service", - "kafka-streams-service", - "topics-and-services", - "multi-file", - "simple-users", - "custom-service-acls", - "custom-user-acls", - "custom-group-id-application", - "custom-group-id-connect", - "custom-application-id-streams", - "custom-storage-topic", - "custom-storage-topics", - "default-replication", - "default-replication-multiple", - "describe-topic-acl-disabled", - "describe-topic-acl-enabled" + "simple", + "application-service", + "kafka-connect-service", + "kafka-streams-service", + "topics-and-services", + "multi-file", + "simple-users", + "custom-service-acls", + "custom-user-acls", + "custom-group-id-application-prefixed", + "custom-group-id-application", + "custom-group-id-connect", + "custom-application-id-streams", + "custom-storage-topic", + "custom-storage-topics", + "default-replication", + "default-replication-multiple", + "describe-topic-acl-disabled", + "describe-topic-acl-enabled" ] } @@ -93,14 +106,14 @@ class PlanCommandIntegrationSpec extends Specification { where: planName << [ - "skip-acls" + "skip-acls" ] } void 'test various valid plans with seed - #planName'() { setup: - TestUtils.cleanUpCluster() - TestUtils.seedCluster() + TestUtils.cleanUpKafkaCluster() + TestUtils.seedKafkaCluster() String planOutputFile = "/tmp/plan.json" String file = TestUtils.getResourceFilePath("plans/${planName}.yaml") MainCommand mainCommand = new MainCommand() @@ -139,8 +152,8 @@ class PlanCommandIntegrationSpec extends Specification { void 'test include unchanged flag - #planNam #includeUnchanged'() { setup: - TestUtils.cleanUpCluster() - TestUtils.seedCluster() + TestUtils.cleanUpKafkaCluster() + TestUtils.seedKafkaCluster() String planOutputFile = "/tmp/plan.json" String file = TestUtils.getResourceFilePath("plans/${planName}.yaml") MainCommand mainCommand = new MainCommand() @@ -196,15 +209,15 @@ class PlanCommandIntegrationSpec extends Specification { where: planName << [ - "invalid-missing-principal", - "invalid-topic", - "unrecognized-property", - "invalid-format", - "invalid-missing-user-principal", - "invalid-storage-topics", - "invalid-default-replication-1", - "invalid-default-replication-2", - "invalid-topic-remove-partitions" + "invalid-missing-principal", + "invalid-topic", + "unrecognized-property", + "invalid-format", + "invalid-missing-user-principal", + "invalid-storage-topics", + "invalid-default-replication-1", + "invalid-default-replication-2", + "invalid-topic-remove-partitions" ] } @@ -256,8 +269,8 @@ class PlanCommandIntegrationSpec extends Specification { void 'test plan that has no changes - #includeUnchanged'() { setup: - TestUtils.cleanUpCluster() - TestUtils.seedCluster() + TestUtils.cleanUpKafkaCluster() + TestUtils.seedKafkaCluster() ByteArrayOutputStream out = new ByteArrayOutputStream() PrintStream oldOut = System.out System.setOut(new PrintStream(out)) @@ -294,4 +307,273 @@ class PlanCommandIntegrationSpec extends Specification { "no-changes" | false "no-changes" | true } + + void 'test various valid schema registry plans - #planName'() { + setup: + TestUtils.cleanUpAll() + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + + then: + exitCode == 0 + + when: + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-plan.json") + + then: + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + + where: + planName << [ + "schema-registry-new-json", + "schema-registry-new-avro", + "schema-registry-new-proto", + "schema-registry-default", + "schema-registry-mix" + ] + } + + void 'test various valid schema registry plans with seed - #planName'() { + setup: + TestUtils.seedSchemaRegistry() + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode + if (deleteDisabled) { + exitCode = cmd.execute("-f", file, "--no-delete", "plan", "-o", planOutputFile) + } else { + exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + } + + then: + exitCode == 0 + + when: + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-plan.json") + + then: + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + + where: + planName | deleteDisabled + "seed-schema-modification" | false + "seed-schema-modification-2" | false + "seed-schema-modification-3" | false + "seed-schema-modification-4" | false + "seed-schema-modification-blacklist" | false + "seed-schema-modification-no-delete" | true + "seed-schema-add-with-reference" | false + } + + void 'test schema registry plan that has no changes - #includeUnchanged'() { + setup: + TestUtils.seedSchemaRegistry() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldOut = System.out + System.setOut(new PrintStream(out)) + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = -1 + if (includeUnchanged) { + exitCode = cmd.execute("-f", file, "plan", "--include-unchanged", "-o", planOutputFile) + } else { + exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + } + + then: + exitCode == 0 + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/no-changes-output.txt") + + when: + String expected = includeUnchanged ? "${planName}-include-unchanged" : planName + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${expected}-plan.json") + + then: + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + + cleanup: + System.setOut(oldOut) + + where: + planName | includeUnchanged + "no-changes" | false + "no-changes" | true + } + + void 'test invalid schema registry plans - #planName'() { + setup: + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan") + + then: + exitCode == 2 + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planName}-output.txt") + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + + where: + planName << [ + "invalid-type", + "invalid-missing-type", + "invalid-compatibility", + "invalid-missing-compatibility", + "invalid-unrecognized-property", + "invalid-missing-file-and-schema", + "invalid-both-file-and-schema" + ] + } + + void 'test various invalid schema registry plans with seed - #planName'() { + setup: + TestUtils.seedSchemaRegistry() + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan") + + then: + exitCode == 2 + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planName}-output.txt") + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + + where: + planName << [ + "invalid-modify-type", + "invalid-modify-compatibility", + "invalid-modify-not-compatible" + ] + } + + void 'test various invalid schema registry plans with seed (regex) - #planName'() { + setup: + TestUtils.seedSchemaRegistry() + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan") + String pattern = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-output.txt") + + then: + exitCode == 2 + out.toString().matches(pattern) + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + + where: + planName << [ + "invalid-reference" + ] + } + + void 'test Specific compatibility plans with manual seed'() { + setup: + CachedSchemaRegistryClient schemaRegistryClient = TestUtils.cachedSchemaRegistryClientRef.get() + TestUtils.createSchema("test-1-avro", SchemaType.AVRO, TestUtils.getResourceFileContent("plans/schema_registry/schemas/schema-registry-schema4.avsc"), + schemaRegistryClient, SchemaCompatibility.BACKWARD) + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/invalid-modify-not-compatible2.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan") + String pattern =TestUtils.getResourceFileContent("plans/schema_registry/invalid-modify-not-compatible2-output.txt") + + then: + exitCode == 2 + out.toString().matches(pattern) + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + } + void 'test Specific for next deferred apply with manual seed'() { + setup: + CachedSchemaRegistryClient schemaRegistryClient = TestUtils.cachedSchemaRegistryClientRef.get() + TestUtils.seedSchemaRegistry() + TestUtils.createSchema("schema-10-json", SchemaType.JSON, "{\"type\":\"object\",\"properties\":{\"test2\":{\"type\":\"string\"}}, \"additionalProperties\": false}", + schemaRegistryClient, SchemaCompatibility.BACKWARD) + List reference = new ArrayList() + reference.add(new SchemaReference("otherschema", "schema-10-json", 1)) + List subjects = schemaRegistryClient.getAllVersions("schema-10-json"); + TestUtils.createSchema("schema-11-json", SchemaType.JSON, '{"type":"object","properties":{"test2":{"\$ref":"otherschema"}}, "additionalProperties": false}', + schemaRegistryClient, SchemaCompatibility.BACKWARD, reference) + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/seed-schema-delete-reference.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/seed-schema-delete-reference-plan.json") + + then: + exitCode == 0 + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy index bfc943a1..019d0a5d 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy @@ -1,19 +1,51 @@ package com.devshawn.kafka.gitops +import java.nio.file.Paths +import java.time.Duration +import java.util.concurrent.atomic.AtomicReference + import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.admin.Admin import org.apache.kafka.clients.admin.NewTopic -import org.apache.kafka.common.acl.* +import org.apache.kafka.common.acl.AccessControlEntry +import org.apache.kafka.common.acl.AccessControlEntryFilter +import org.apache.kafka.common.acl.AclBinding +import org.apache.kafka.common.acl.AclBindingFilter +import org.apache.kafka.common.acl.AclOperation +import org.apache.kafka.common.acl.AclPermissionType import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.resource.PatternType import org.apache.kafka.common.resource.ResourcePattern import org.apache.kafka.common.resource.ResourcePatternFilter import org.apache.kafka.common.resource.ResourceType -import spock.util.concurrent.PollingConditions -import java.nio.file.Paths +import com.devshawn.kafka.gitops.config.SchemaRegistryConfig +import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader +import com.devshawn.kafka.gitops.enums.SchemaCompatibility +import com.devshawn.kafka.gitops.enums.SchemaType +import com.devshawn.kafka.gitops.service.SchemaRegistryService + +import io.confluent.kafka.schemaregistry.AbstractSchemaProvider +import io.confluent.kafka.schemaregistry.ParsedSchema +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.rest.RestService +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException +import spock.util.concurrent.PollingConditions class TestUtils { + public static final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>() + + static { + Map config = SchemaRegistryConfigLoader.load().getConfig() + RestService restService = new RestService(config.get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()) + CachedSchemaRegistryClient cachedSchemaRegistryClient = new CachedSchemaRegistryClient(restService, 1, config) + cachedSchemaRegistryClientRef.set(cachedSchemaRegistryClient) + } + + private TestUtils() { + } static String getFileContent(String fileName) { File file = new File(fileName) @@ -32,38 +64,94 @@ class TestUtils { return file.getAbsolutePath() } - static void cleanUpCluster() { - def conditions = new PollingConditions(timeout: 60, initialDelay: 2, factor: 1.25) + static void cleanUpSchemaRegistry() { + try { + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get() + Collection subjects = schemaRegistryClient.getAllSubjects() + while (subjects.size() > 0) { + for (subject in subjects) { + try { + println "Deleting " +subject + schemaRegistryClient.deleteSubject(subject) + schemaRegistryClient.deleteSubject(subject, true) + } catch (RestClientException ex) { + if(ex.errorCode == 42206) { + println "Error cleaning referenced schema (" +subject + ")... will do it later" + } else { + throw ex + } + } + } + subjects = schemaRegistryClient.getAllSubjects() + Collections.shuffle(subjects) + } + println "Finished cleaning up schema registry" + } catch (Exception ex) { + println "Error cleaning up schema registry" + throw new RuntimeException("Error cleaning up schema registry", ex) + } + } + static void seedSchemaRegistry() { try { - AdminClient adminClient = AdminClient.create(getKafkaConfig()) + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get() + createSchema("schema-1-json", SchemaType.JSON, + "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}", schemaRegistryClient, SchemaCompatibility.BACKWARD) + createSchema("schema-2-avro", SchemaType.AVRO, + "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + schemaRegistryClient, SchemaCompatibility.BACKWARD) + createSchema("schema-3-protobuf", SchemaType.PROTOBUF, + "syntax = \"proto3\";\npackage com.acme;\n\nmessage OtherRecord {\n int32 an_id = 1;\n}\n", + schemaRegistryClient, SchemaCompatibility.FULL) + + println "Finished seeding schema registry" + } catch (Exception ex) { + println "Error seeding up Schema registry" + throw new RuntimeException("Error seeding up schema registry", ex) + } + } + + static void cleanUpAll() { + cleanUpKafkaCluster() + cleanUpSchemaRegistry() + } + + static void cleanUpKafkaCluster() { + def conditions = new PollingConditions(timeout: 30, initialDelay: 2, delay: 2) + + try { + Admin adminClient = Admin.create(getKafkaConfig()) Set topics = adminClient.listTopics().names().get() - adminClient.deleteTopics(topics).all().get() - + // Do not remove the schema registry topic + topics.remove("_schemas") + adminClient.deleteTopics(topics) conditions.eventually { + println "Testing if kafka topics still exist..." Set remainingTopics = adminClient.listTopics().names().get() - assert remainingTopics.size() == 0 + assert remainingTopics.size() == 1 + assert remainingTopics.getAt(0).equals("_schemas") } - + AclBindingFilter filter = getWildcardFilter() adminClient.deleteAcls(Collections.singletonList(filter)) conditions.eventually { + println "Testing if kafka acls still exist..." List acls = new ArrayList<>(adminClient.describeAcls(filter).values().get()) assert acls.size() == 0 } - println "Finished cleaning up cluster" + adminClient.close(Duration.ofSeconds(10)); + println "Finished cleaning up kafka cluster" } catch (Exception ex) { println "Error cleaning up kafka cluster" - println ex + throw new RuntimeException("Error cleaning up kafka cluster", ex) } - } - static void seedCluster() { + static void seedKafkaCluster() { def conditions = new PollingConditions(timeout: 60, initialDelay: 2, factor: 1.25) try { - AdminClient adminClient = AdminClient.create(getKafkaConfig()) + Admin adminClient = Admin.create(getKafkaConfig()) createTopic("delete-topic", 1, adminClient) createTopic("test-topic", 1, adminClient) createTopic("topic-with-configs-1", 3, adminClient, ["cleanup.policy": "compact", "segment.bytes": "100000"]) @@ -72,23 +160,43 @@ class TestUtils { conditions.eventually { Set newTopics = adminClient.listTopics().names().get() - assert newTopics.size() == 4 + assert newTopics.size() == 5 List newAcls = new ArrayList<>(adminClient.describeAcls(getWildcardFilter()).values().get()) assert newAcls.size() == 1 } + adminClient.close() println "Finished seeding kafka cluster" } catch (Exception ex) { println "Error seeding up kafka cluster" - ex.printStackTrace() + throw new RuntimeException("Error seeding up kafka cluster", ex) } } - static void createTopic(String name, int partitions, AdminClient adminClient) { + static void createSchema(String subject, SchemaType type, String schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { + AbstractSchemaProvider schemaProvider = new SchemaRegistryService(SchemaRegistryConfigLoader.load()).schemaProviderFromType(type) + ParsedSchema parsedSchema = schemaProvider.parseSchema(schema, Collections.emptyList()).get() + internalCreateSchema(subject, type, parsedSchema, client, compatibility) + } + + static void createSchema(String subject, SchemaType type, String schema , SchemaRegistryClient client, SchemaCompatibility compatibility, List references) { + AbstractSchemaProvider schemaProvider = new SchemaRegistryService(SchemaRegistryConfigLoader.load()).schemaProviderFromType(type) + ParsedSchema parsedSchema = schemaProvider.parseSchema(schema, references).get() + internalCreateSchema(subject, type, parsedSchema, client, compatibility) + } + + private static void internalCreateSchema(String subject, SchemaType type, ParsedSchema schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get() + int id = schemaRegistryClient.register(subject, schema) + String compat = schemaRegistryClient.updateCompatibility(subject, compatibility.toString()) + println "Schema subject '" + subject + "' with id " + id + " created (compatibility: " + compat + ")" + } + + static void createTopic(String name, int partitions, Admin adminClient) { createTopic(name, partitions, adminClient, null) } - static void createTopic(String name, int partitions, AdminClient adminClient, Map configs) { + static void createTopic(String name, int partitions, Admin adminClient, Map configs) { NewTopic newTopic = new NewTopic(name, partitions, (short) 2) if (configs != null) { newTopic.configs(configs) @@ -96,7 +204,7 @@ class TestUtils { adminClient.createTopics(Collections.singletonList(newTopic)).all().get() } - static void createAcl(AdminClient adminClient) { + static void createAcl(Admin adminClient) { ResourcePattern resourcePattern = new ResourcePattern(ResourceType.TOPIC, "test-topic", PatternType.LITERAL) AccessControlEntry accessControlEntry = new AccessControlEntry("User:test", "*", AclOperation.READ, AclPermissionType.ALLOW) AclBinding aclBinding = new AclBinding(resourcePattern, accessControlEntry) @@ -113,10 +221,10 @@ class TestUtils { String jaasConfig = String.format("org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";", System.getenv("KAFKA_SASL_JAAS_USERNAME"), System.getenv("KAFKA_SASL_JAAS_PASSWORD")) return [ - (CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG): System.getenv("KAFKA_BOOTSTRAP_SERVERS"), - (CommonClientConfigs.SECURITY_PROTOCOL_CONFIG): System.getenv("KAFKA_SECURITY_PROTOCOL"), - (SaslConfigs.SASL_MECHANISM) : System.getenv("KAFKA_SASL_MECHANISM"), - (SaslConfigs.SASL_JAAS_CONFIG) : jaasConfig, + (CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG): System.getenv("KAFKA_BOOTSTRAP_SERVERS"), + (CommonClientConfigs.SECURITY_PROTOCOL_CONFIG): System.getenv("KAFKA_SECURITY_PROTOCOL"), + (SaslConfigs.SASL_MECHANISM) : System.getenv("KAFKA_SASL_MECHANISM"), + (SaslConfigs.SASL_JAAS_CONFIG) : jaasConfig, ] } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy index a949286f..bc9e4df3 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy @@ -14,7 +14,7 @@ class KafkaGitopsConfigLoaderSpec extends Specification { EnvironmentVariables environmentVariables void setupSpec() { - environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092") + environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092,localhost:9093,localhost:9094") environmentVariables.set("KAFKA_SASL_MECHANISM", "PLAIN") environmentVariables.set("KAFKA_SECURITY_PROTOCOL", "SASL_PLAINTEXT") } @@ -54,7 +54,7 @@ class KafkaGitopsConfigLoaderSpec extends Specification { then: config.config.get(CommonClientConfigs.CLIENT_ID_CONFIG) == "kafka-gitops" - config.config.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG) == "localhost:9092" + config.config.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG) == "localhost:9092,localhost:9093,localhost:9094" config.config.get(SaslConfigs.SASL_MECHANISM) == "PLAIN" } diff --git a/src/test/resources/plans/application-service-plan.json b/src/test/resources/plans/application-service-plan.json index 72daf4d2..7c87d536 100644 --- a/src/test/resources/plans/application-service-plan.json +++ b/src/test/resources/plans/application-service-plan.json @@ -1,44 +1,45 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/application-service.yaml b/src/test/resources/plans/application-service.yaml index f9615d40..6367d57e 100644 --- a/src/test/resources/plans/application-service.yaml +++ b/src/test/resources/plans/application-service.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: test-service: type: application diff --git a/src/test/resources/plans/custom-application-id-streams-plan.json b/src/test/resources/plans/custom-application-id-streams-plan.json index 839e0622..f313955f 100644 --- a/src/test/resources/plans/custom-application-id-streams-plan.json +++ b/src/test/resources/plans/custom-application-id-streams-plan.json @@ -1,187 +1,188 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "streams-application-0", - "aclDetails": { - "name": "my-other-streams-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-0", + "aclDetails" : { + "name" : "my-other-streams-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-1", - "aclDetails": { - "name": "my-streams-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-1", + "aclDetails" : { + "name" : "my-streams-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-2", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-2", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-3", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-3", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-4", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-4", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-5", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-5", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-6", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "CREATE", - "permission": "ALLOW" + "name" : "streams-application-6", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "CREATE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-7", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER", - "permission": "ALLOW" + "name" : "streams-application-7", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-8", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-8", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-9", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-9", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-10", - "aclDetails": { - "name": "test-streams", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-10", + "aclDetails" : { + "name" : "test-streams", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-11", - "aclDetails": { - "name": "test-streams", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-11", + "aclDetails" : { + "name" : "test-streams", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-12", - "aclDetails": { - "name": "test-streams", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-12", + "aclDetails" : { + "name" : "test-streams", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-13", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-13", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-application-id-streams.yaml b/src/test/resources/plans/custom-application-id-streams.yaml index 57cc60e5..5ff9b2a4 100644 --- a/src/test/resources/plans/custom-application-id-streams.yaml +++ b/src/test/resources/plans/custom-application-id-streams.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: streams-application: type: kafka-streams diff --git a/src/test/resources/plans/custom-group-id-application-plan.json b/src/test/resources/plans/custom-group-id-application-plan.json index 8db50dae..c3a9b966 100644 --- a/src/test/resources/plans/custom-group-id-application-plan.json +++ b/src/test/resources/plans/custom-group-id-application-plan.json @@ -1,44 +1,45 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-group-id-application-prefixed-apply-output.txt b/src/test/resources/plans/custom-group-id-application-prefixed-apply-output.txt new file mode 100644 index 00000000..4c4b3359 --- /dev/null +++ b/src/test/resources/plans/custom-group-id-application-prefixed-apply-output.txt @@ -0,0 +1,45 @@ +Executing apply... + +Applying: [CREATE] + ++ [ACL] test-service-0 + + resource_name: test-topic + + resource_type: TOPIC + + resource_pattern: LITERAL + + resource_principal: User:test + + host: * + + operation: WRITE + + permission: ALLOW + + +Successfully applied. + +Applying: [CREATE] + ++ [ACL] test-service-1 + + resource_name: another-test-topic + + resource_type: TOPIC + + resource_pattern: LITERAL + + resource_principal: User:test + + host: * + + operation: READ + + permission: ALLOW + + +Successfully applied. + +Applying: [CREATE] + ++ [ACL] test-service-2 + + resource_name: test-service-application-prefixed + + resource_type: GROUP + + resource_pattern: PREFIXED + + resource_principal: User:test + + host: * + + operation: READ + + permission: ALLOW + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 3 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/custom-group-id-application-prefixed-plan.json b/src/test/resources/plans/custom-group-id-application-prefixed-plan.json new file mode 100644 index 00000000..8f385130 --- /dev/null +++ b/src/test/resources/plans/custom-group-id-application-prefixed-plan.json @@ -0,0 +1,45 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ + { + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "test-service-1", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service-application-prefixed", + "type" : "GROUP", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/custom-group-id-application-prefixed.yaml b/src/test/resources/plans/custom-group-id-application-prefixed.yaml new file mode 100644 index 00000000..4bd84c3e --- /dev/null +++ b/src/test/resources/plans/custom-group-id-application-prefixed.yaml @@ -0,0 +1,14 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas +services: + test-service: + type: application + group-id: test-service-application-prefixed* + principal: User:test + produces: + - test-topic + consumes: + - another-test-topic \ No newline at end of file diff --git a/src/test/resources/plans/custom-group-id-application.yaml b/src/test/resources/plans/custom-group-id-application.yaml index f3be06cc..3e52fa18 100644 --- a/src/test/resources/plans/custom-group-id-application.yaml +++ b/src/test/resources/plans/custom-group-id-application.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: test-service: type: application diff --git a/src/test/resources/plans/custom-group-id-connect-plan.json b/src/test/resources/plans/custom-group-id-connect-plan.json index 579b8078..f97a438d 100644 --- a/src/test/resources/plans/custom-group-id-connect-plan.json +++ b/src/test/resources/plans/custom-group-id-connect-plan.json @@ -1,135 +1,136 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-connect-cluster-0", - "aclDetails": { - "name": "connect-configs-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-0", + "aclDetails" : { + "name" : "connect-configs-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-1", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-1", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-2", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-2", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-3", - "aclDetails": { - "name": "connect-configs-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-3", + "aclDetails" : { + "name" : "connect-configs-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-4", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-4", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-5", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-5", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-6", - "aclDetails": { - "name": "connect-cluster", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-6", + "aclDetails" : { + "name" : "connect-cluster", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-7", - "aclDetails": { - "name": "production-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-7", + "aclDetails" : { + "name" : "production-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-8", - "aclDetails": { - "name": "consumption-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-8", + "aclDetails" : { + "name" : "consumption-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-9", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-9", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-group-id-connect.yaml b/src/test/resources/plans/custom-group-id-connect.yaml index 11e6e57e..c5bea388 100644 --- a/src/test/resources/plans/custom-group-id-connect.yaml +++ b/src/test/resources/plans/custom-group-id-connect.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/custom-service-acls-plan.json b/src/test/resources/plans/custom-service-acls-plan.json index d20095ea..1112b9f3 100644 --- a/src/test/resources/plans/custom-service-acls-plan.json +++ b/src/test/resources/plans/custom-service-acls-plan.json @@ -1,18 +1,19 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "kafka.", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "kafka.", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-service-acls.yaml b/src/test/resources/plans/custom-service-acls.yaml index 31e4a26e..6112627c 100644 --- a/src/test/resources/plans/custom-service-acls.yaml +++ b/src/test/resources/plans/custom-service-acls.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: test-service: type: application diff --git a/src/test/resources/plans/custom-storage-topic-plan.json b/src/test/resources/plans/custom-storage-topic-plan.json index fcdc8f97..8b87ee09 100644 --- a/src/test/resources/plans/custom-storage-topic-plan.json +++ b/src/test/resources/plans/custom-storage-topic-plan.json @@ -1,135 +1,136 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-connect-cluster-0", - "aclDetails": { - "name": "test-connect-cluster-configs", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-0", + "aclDetails" : { + "name" : "test-connect-cluster-configs", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-1", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-1", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-2", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-2", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-3", - "aclDetails": { - "name": "test-connect-cluster-configs", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-3", + "aclDetails" : { + "name" : "test-connect-cluster-configs", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-4", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-4", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-5", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-5", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-6", - "aclDetails": { - "name": "test-connect-cluster", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-6", + "aclDetails" : { + "name" : "test-connect-cluster", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-7", - "aclDetails": { - "name": "production-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-7", + "aclDetails" : { + "name" : "production-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-8", - "aclDetails": { - "name": "consumption-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-8", + "aclDetails" : { + "name" : "consumption-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-9", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-9", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-storage-topic.yaml b/src/test/resources/plans/custom-storage-topic.yaml index 57df46fc..3c7b5d04 100644 --- a/src/test/resources/plans/custom-storage-topic.yaml +++ b/src/test/resources/plans/custom-storage-topic.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/custom-storage-topics-plan.json b/src/test/resources/plans/custom-storage-topics-plan.json index 5d25fadf..e4d78f5b 100644 --- a/src/test/resources/plans/custom-storage-topics-plan.json +++ b/src/test/resources/plans/custom-storage-topics-plan.json @@ -1,135 +1,136 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-connect-cluster-0", - "aclDetails": { - "name": "config-custom-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-0", + "aclDetails" : { + "name" : "config-custom-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-1", - "aclDetails": { - "name": "offset-topic-custom", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-1", + "aclDetails" : { + "name" : "offset-topic-custom", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-2", - "aclDetails": { - "name": "custom-status-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-2", + "aclDetails" : { + "name" : "custom-status-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-3", - "aclDetails": { - "name": "config-custom-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-3", + "aclDetails" : { + "name" : "config-custom-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-4", - "aclDetails": { - "name": "offset-topic-custom", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-4", + "aclDetails" : { + "name" : "offset-topic-custom", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-5", - "aclDetails": { - "name": "custom-status-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-5", + "aclDetails" : { + "name" : "custom-status-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-6", - "aclDetails": { - "name": "test-connect-cluster", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-6", + "aclDetails" : { + "name" : "test-connect-cluster", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-7", - "aclDetails": { - "name": "production-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-7", + "aclDetails" : { + "name" : "production-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-8", - "aclDetails": { - "name": "consumption-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-8", + "aclDetails" : { + "name" : "consumption-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-9", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-9", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-storage-topics.yaml b/src/test/resources/plans/custom-storage-topics.yaml index 0a56bd64..f799e2e9 100644 --- a/src/test/resources/plans/custom-storage-topics.yaml +++ b/src/test/resources/plans/custom-storage-topics.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/custom-user-acls-plan.json b/src/test/resources/plans/custom-user-acls-plan.json index 9c8eba5c..a0af374e 100644 --- a/src/test/resources/plans/custom-user-acls-plan.json +++ b/src/test/resources/plans/custom-user-acls-plan.json @@ -1,18 +1,19 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-user-0", - "aclDetails": { - "name": "kafka.", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-user-0", + "aclDetails" : { + "name" : "kafka.", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-user-acls.yaml b/src/test/resources/plans/custom-user-acls.yaml index caf671ab..2fd46d64 100644 --- a/src/test/resources/plans/custom-user-acls.yaml +++ b/src/test/resources/plans/custom-user-acls.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + users: test-user: principal: User:test diff --git a/src/test/resources/plans/default-replication-multiple-plan.json b/src/test/resources/plans/default-replication-multiple-plan.json index 5c816376..8fbd5389 100644 --- a/src/test/resources/plans/default-replication-multiple-plan.json +++ b/src/test/resources/plans/default-replication-multiple-plan.json @@ -1,44 +1,45 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 3, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 3, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "another-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "another-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "last-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 4, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "last-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 4, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/default-replication-multiple.yaml b/src/test/resources/plans/default-replication-multiple.yaml index 05aee33d..08192a03 100644 --- a/src/test/resources/plans/default-replication-multiple.yaml +++ b/src/test/resources/plans/default-replication-multiple.yaml @@ -2,7 +2,9 @@ settings: topics: defaults: replication: 3 - + blacklist: + prefixed: + - _schemas topics: test-topic: partitions: 6 diff --git a/src/test/resources/plans/default-replication-plan.json b/src/test/resources/plans/default-replication-plan.json index e841e1a1..03744be7 100644 --- a/src/test/resources/plans/default-replication-plan.json +++ b/src/test/resources/plans/default-replication-plan.json @@ -1,18 +1,19 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/default-replication.yaml b/src/test/resources/plans/default-replication.yaml index fc5932b9..09f48932 100644 --- a/src/test/resources/plans/default-replication.yaml +++ b/src/test/resources/plans/default-replication.yaml @@ -2,7 +2,9 @@ settings: topics: defaults: replication: 2 - + blacklist: + prefixed: + - _schemas topics: test-topic: partitions: 6 diff --git a/src/test/resources/plans/describe-topic-acl-disabled-plan.json b/src/test/resources/plans/describe-topic-acl-disabled-plan.json index 130ee3cc..4972b923 100644 --- a/src/test/resources/plans/describe-topic-acl-disabled-plan.json +++ b/src/test/resources/plans/describe-topic-acl-disabled-plan.json @@ -1,369 +1,370 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "normal-application-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "normal-application-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "normal-application-1", - "aclDetails": { - "name": "first-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "normal-application-1", + "aclDetails" : { + "name" : "first-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "normal-application-2", - "aclDetails": { - "name": "normal-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "normal-application-2", + "aclDetails" : { + "name" : "normal-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-0", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-0", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-1", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-1", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-2", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-2", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-3", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-3", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-4", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-4", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-5", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-5", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-6", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "CREATE", - "permission": "ALLOW" + "name" : "streams-application-6", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "CREATE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-7", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER", - "permission": "ALLOW" + "name" : "streams-application-7", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-8", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-8", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-9", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-9", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-10", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-10", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-11", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-11", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-12", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-12", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-13", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-13", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-0", - "aclDetails": { - "name": "poison-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-0", + "aclDetails" : { + "name" : "poison-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-1", - "aclDetails": { - "name": "connect-configs-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-1", + "aclDetails" : { + "name" : "connect-configs-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-2", - "aclDetails": { - "name": "connect-offsets-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-2", + "aclDetails" : { + "name" : "connect-offsets-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-3", - "aclDetails": { - "name": "connect-status-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-3", + "aclDetails" : { + "name" : "connect-status-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-4", - "aclDetails": { - "name": "connect-configs-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-4", + "aclDetails" : { + "name" : "connect-configs-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-5", - "aclDetails": { - "name": "connect-offsets-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-5", + "aclDetails" : { + "name" : "connect-offsets-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-6", - "aclDetails": { - "name": "connect-status-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-6", + "aclDetails" : { + "name" : "connect-status-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-7", - "aclDetails": { - "name": "kafka-connect-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-7", + "aclDetails" : { + "name" : "kafka-connect-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-8", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-8", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-9", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-9", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-10", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-10", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/describe-topic-acl-disabled.yaml b/src/test/resources/plans/describe-topic-acl-disabled.yaml index c93e625a..2adf1d8d 100644 --- a/src/test/resources/plans/describe-topic-acl-disabled.yaml +++ b/src/test/resources/plans/describe-topic-acl-disabled.yaml @@ -2,7 +2,10 @@ settings: services: acls: describeTopicEnabled: false - + topics: + blacklist: + prefixed: + - _schemas services: normal-application: type: application diff --git a/src/test/resources/plans/describe-topic-acl-enabled-plan.json b/src/test/resources/plans/describe-topic-acl-enabled-plan.json index 689b778c..724e7c10 100644 --- a/src/test/resources/plans/describe-topic-acl-enabled-plan.json +++ b/src/test/resources/plans/describe-topic-acl-enabled-plan.json @@ -1,460 +1,461 @@ { - "topicPlans": [], - "aclPlans": [ - { - "name": "normal-application-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "normal-application-1", - "aclDetails": { - "name": "first-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "normal-application-2", - "aclDetails": { - "name": "first-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "normal-application-3", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "normal-application-4", - "aclDetails": { - "name": "normal-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-0", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-1", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-2", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-3", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-4", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-5", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-6", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-7", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-8", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "CREATE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-9", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-10", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER_CONFIGS", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-11", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-12", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-13", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-14", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-15", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-0", - "aclDetails": { - "name": "poison-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-1", - "aclDetails": { - "name": "poison-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-2", - "aclDetails": { - "name": "connect-configs-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-3", - "aclDetails": { - "name": "connect-offsets-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-4", - "aclDetails": { - "name": "connect-status-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-5", - "aclDetails": { - "name": "connect-configs-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-6", - "aclDetails": { - "name": "connect-offsets-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-7", - "aclDetails": { - "name": "connect-status-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-8", - "aclDetails": { - "name": "kafka-connect-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-9", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-10", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-11", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-12", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-13", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ + { + "name" : "normal-application-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "normal-application-1", + "aclDetails" : { + "name" : "first-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "normal-application-2", + "aclDetails" : { + "name" : "first-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "normal-application-3", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "normal-application-4", + "aclDetails" : { + "name" : "normal-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-0", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-1", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-2", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-3", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-4", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-5", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-6", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-7", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-8", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "CREATE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-9", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-10", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER_CONFIGS", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-11", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-12", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-13", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-14", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-15", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-0", + "aclDetails" : { + "name" : "poison-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-1", + "aclDetails" : { + "name" : "poison-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-2", + "aclDetails" : { + "name" : "connect-configs-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-3", + "aclDetails" : { + "name" : "connect-offsets-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-4", + "aclDetails" : { + "name" : "connect-status-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-5", + "aclDetails" : { + "name" : "connect-configs-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-6", + "aclDetails" : { + "name" : "connect-offsets-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-7", + "aclDetails" : { + "name" : "connect-status-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-8", + "aclDetails" : { + "name" : "kafka-connect-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-9", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-10", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-11", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-12", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-13", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/describe-topic-acl-enabled.yaml b/src/test/resources/plans/describe-topic-acl-enabled.yaml index d19d782d..27a50cf0 100644 --- a/src/test/resources/plans/describe-topic-acl-enabled.yaml +++ b/src/test/resources/plans/describe-topic-acl-enabled.yaml @@ -2,7 +2,10 @@ settings: services: acls: describeTopicEnabled: true - + topics: + blacklist: + prefixed: + - _schemas services: normal-application: type: application diff --git a/src/test/resources/plans/invalid-custom-user-acls-1.yaml b/src/test/resources/plans/invalid-custom-user-acls-1.yaml index 9aa74b00..1382f6c0 100644 --- a/src/test/resources/plans/invalid-custom-user-acls-1.yaml +++ b/src/test/resources/plans/invalid-custom-user-acls-1.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + users: test-user: principal: User:test diff --git a/src/test/resources/plans/invalid-plan.json b/src/test/resources/plans/invalid-plan.json index 96833b5b..77685ada 100644 --- a/src/test/resources/plans/invalid-plan.json +++ b/src/test/resources/plans/invalid-plan.json @@ -1,3 +1,3 @@ { - "topicPlans": {} + "topicPlans" : {} } \ No newline at end of file diff --git a/src/test/resources/plans/invalid-storage-topics.yaml b/src/test/resources/plans/invalid-storage-topics.yaml index 9f2833a6..8d42e298 100644 --- a/src/test/resources/plans/invalid-storage-topics.yaml +++ b/src/test/resources/plans/invalid-storage-topics.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/invalid-topic-remove-partitions.yaml b/src/test/resources/plans/invalid-topic-remove-partitions.yaml index c5af5b16..b0214936 100644 --- a/src/test/resources/plans/invalid-topic-remove-partitions.yaml +++ b/src/test/resources/plans/invalid-topic-remove-partitions.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/invalid-topic.yaml b/src/test/resources/plans/invalid-topic.yaml index 82e7e7e7..22d538fd 100644 --- a/src/test/resources/plans/invalid-topic.yaml +++ b/src/test/resources/plans/invalid-topic.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: replication: 2 \ No newline at end of file diff --git a/src/test/resources/plans/kafka-connect-service-plan.json b/src/test/resources/plans/kafka-connect-service-plan.json index 14d019fe..50068b8e 100644 --- a/src/test/resources/plans/kafka-connect-service-plan.json +++ b/src/test/resources/plans/kafka-connect-service-plan.json @@ -1,135 +1,136 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-connect-cluster-0", - "aclDetails": { - "name": "connect-configs-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-0", + "aclDetails" : { + "name" : "connect-configs-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-1", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-1", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-2", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-2", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-3", - "aclDetails": { - "name": "connect-configs-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-3", + "aclDetails" : { + "name" : "connect-configs-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-4", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-4", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-5", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-5", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-6", - "aclDetails": { - "name": "test-connect-cluster", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-6", + "aclDetails" : { + "name" : "test-connect-cluster", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-7", - "aclDetails": { - "name": "production-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-7", + "aclDetails" : { + "name" : "production-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-8", - "aclDetails": { - "name": "consumption-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-8", + "aclDetails" : { + "name" : "consumption-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-9", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-9", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/kafka-connect-service.yaml b/src/test/resources/plans/kafka-connect-service.yaml index c7690046..b38613f4 100644 --- a/src/test/resources/plans/kafka-connect-service.yaml +++ b/src/test/resources/plans/kafka-connect-service.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/kafka-streams-service-plan.json b/src/test/resources/plans/kafka-streams-service-plan.json index c752cd52..d2145b9a 100644 --- a/src/test/resources/plans/kafka-streams-service-plan.json +++ b/src/test/resources/plans/kafka-streams-service-plan.json @@ -1,200 +1,201 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "streams-application-0", - "aclDetails": { - "name": "my-other-streams-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-0", + "aclDetails" : { + "name" : "my-other-streams-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-1", - "aclDetails": { - "name": "another-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-1", + "aclDetails" : { + "name" : "another-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-2", - "aclDetails": { - "name": "my-streams-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-2", + "aclDetails" : { + "name" : "my-streams-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-3", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-3", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-4", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-4", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-5", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-5", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-6", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-6", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-7", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "CREATE", - "permission": "ALLOW" + "name" : "streams-application-7", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "CREATE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-8", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER", - "permission": "ALLOW" + "name" : "streams-application-8", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-9", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-9", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-10", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-10", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-11", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-11", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-12", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-12", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-13", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-13", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-14", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-14", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/kafka-streams-service.yaml b/src/test/resources/plans/kafka-streams-service.yaml index 3c408bb0..11fc9787 100644 --- a/src/test/resources/plans/kafka-streams-service.yaml +++ b/src/test/resources/plans/kafka-streams-service.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: streams-application: type: kafka-streams diff --git a/src/test/resources/plans/multi-file-plan.json b/src/test/resources/plans/multi-file-plan.json index 0a294973..2a654f18 100644 --- a/src/test/resources/plans/multi-file-plan.json +++ b/src/test/resources/plans/multi-file-plan.json @@ -1,58 +1,59 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/multi-file.yaml b/src/test/resources/plans/multi-file.yaml index 62c6b1ea..9ff52700 100644 --- a/src/test/resources/plans/multi-file.yaml +++ b/src/test/resources/plans/multi-file.yaml @@ -3,3 +3,7 @@ settings: services: multi-file-services.yaml topics: multi-file-topics.yaml users: multi-file-users.yaml + topics: + blacklist: + prefixed: + - _schemas diff --git a/src/test/resources/plans/no-changes-include-unchanged-plan.json b/src/test/resources/plans/no-changes-include-unchanged-plan.json index d3bd7798..f667e7a8 100644 --- a/src/test/resources/plans/no-changes-include-unchanged-plan.json +++ b/src/test/resources/plans/no-changes-include-unchanged-plan.json @@ -1,91 +1,92 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "delete-topic", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "delete-topic", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "test-topic", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "test-topic", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-1", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "NO_CHANGE" }, { - "key": "segment.bytes", - "value": "100000", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "segment.bytes", + "value" : "100000", + "previousValue" : null, + "action" : "NO_CHANGE" } ] }, { - "name": "topic-with-configs-2", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-2", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "NO_CHANGE" } ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "NO_CHANGE" + "action" : "NO_CHANGE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/no-changes-plan.json b/src/test/resources/plans/no-changes-plan.json index b93dada0..6dbd87bc 100644 --- a/src/test/resources/plans/no-changes-plan.json +++ b/src/test/resources/plans/no-changes-plan.json @@ -1,4 +1,5 @@ { - "topicPlans": [], - "aclPlans": [] + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/no-changes.yaml b/src/test/resources/plans/no-changes.yaml index c4b2de14..56f672d0 100644 --- a/src/test/resources/plans/no-changes.yaml +++ b/src/test/resources/plans/no-changes.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/schema_registry/invalid-both-file-and-schema-output.txt b/src/test/resources/plans/schema_registry/invalid-both-file-and-schema-output.txt new file mode 100644 index 00000000..215c520d --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-both-file-and-schema-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] schema and file fields cannot be both set at the same time in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-both-file-and-schema.yaml b/src/test/resources/plans/schema_registry/invalid-both-file-and-schema.yaml new file mode 100644 index 00000000..60134094 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-both-file-and-schema.yaml @@ -0,0 +1,12 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: NONE + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + file: "schema-registry-schema1.json" diff --git a/src/test/resources/plans/schema_registry/invalid-compatibility-output.txt b/src/test/resources/plans/schema_registry/invalid-compatibility-output.txt new file mode 100644 index 00000000..057c9a6b --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-compatibility-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Value 'XXX' is not a valid format for: [compatibility] in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-compatibility.yaml b/src/test/resources/plans/schema_registry/invalid-compatibility.yaml new file mode 100644 index 00000000..a5fadab2 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-compatibility.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: XXX + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" diff --git a/src/test/resources/plans/schema_registry/invalid-missing-compatibility-output.txt b/src/test/resources/plans/schema_registry/invalid-missing-compatibility-output.txt new file mode 100644 index 00000000..50175cdb --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-compatibility-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Not set: [compatibility] in state file definition: schema -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-missing-compatibility.yaml b/src/test/resources/plans/schema_registry/invalid-missing-compatibility.yaml new file mode 100644 index 00000000..3aa63b19 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-compatibility.yaml @@ -0,0 +1,10 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" diff --git a/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema-output.txt b/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema-output.txt new file mode 100644 index 00000000..6b6d8665 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] schema or file field must be provided in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema.yaml b/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema.yaml new file mode 100644 index 00000000..46d1dbd1 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema.yaml @@ -0,0 +1,10 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt b/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt new file mode 100644 index 00000000..ad6a0f38 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Not set: [type] in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-missing-type.yaml b/src/test/resources/plans/schema_registry/invalid-missing-type.yaml new file mode 100644 index 00000000..c9c952fc --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-type.yaml @@ -0,0 +1,10 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" diff --git a/src/test/resources/plans/schema_registry/invalid-modify-compatibility-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-compatibility-output.txt new file mode 100644 index 00000000..5b11b4af --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-compatibility-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Changing the subject compatibility is not allowed with kafka-gitops(subject: schema-1-json, current compatibilty: BACKWARD, new compatibilty:FORWARD) diff --git a/src/test/resources/plans/schema_registry/invalid-modify-compatibility.yaml b/src/test/resources/plans/schema_registry/invalid-modify-compatibility.yaml new file mode 100644 index 00000000..179d921c --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-compatibility.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: FORWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt new file mode 100644 index 00000000..8e79b8e4 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] JSON schema 'schema-1-json' is incompatible with an earlier schema: [Found incompatible change: Difference{jsonPath='#/properties/f1', type=PROPERTY_REMOVED_FROM_CLOSED_CONTENT_MODEL}] diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible.yaml b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible.yaml new file mode 100644 index 00000000..d99d0a99 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f2\":{\"type\":\"string\"}}, \"additionalProperties\": false}" \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2-output.txt new file mode 100644 index 00000000..7ba4b7d6 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2-output.txt @@ -0,0 +1 @@ +(?s).*\[INVALID\] AVRO schema 'test-1-avro' is incompatible with an earlier schema: \[Incompatibility\{type:TYPE_MISMATCH, location:\/fields\/7\/type, message:reader type: INT not compatible with writer type: STRING, reader:\{"type":"int","logicalType":"date"\}, writer:"string"\}\].* \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2.yaml b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2.yaml new file mode 100644 index 00000000..81add389 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + test-1-avro: + type: AVRO + compatibility: BACKWARD + file: "schema-registry-schema5.avsc" diff --git a/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt new file mode 100644 index 00000000..0280804e --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Changing the schema type is not allowed (subject: schema-1-json, current type: JSON, new type:AVRO) diff --git a/src/test/resources/plans/schema_registry/invalid-modify-type.yaml b/src/test/resources/plans/schema_registry/invalid-modify-type.yaml new file mode 100644 index 00000000..d6a79471 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-type.yaml @@ -0,0 +1,25 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-2-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-1-json: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" + schema-3-protobuf: + type: PROTOBUF + compatibility: FULL + schema: 'syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +} +' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-reference-output.txt b/src/test/resources/plans/schema_registry/invalid-reference-output.txt new file mode 100644 index 00000000..9372f7f1 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-reference-output.txt @@ -0,0 +1 @@ +(?s).*Version 10 not found\..*\[INVALID\] JSON referenced schema could not be parsed for subject schema-2-json diff --git a/src/test/resources/plans/schema_registry/invalid-reference.yaml b/src/test/resources/plans/schema_registry/invalid-reference.yaml new file mode 100644 index 00000000..95d0180a --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-reference.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-2-json: + type: JSON + compatibility: FORWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}}, \"additionalProperties\": false}" + references: + - name: otherschema + subject: schema-1-json + version: 10 \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-type-output.txt b/src/test/resources/plans/schema_registry/invalid-type-output.txt new file mode 100644 index 00000000..cb65ae27 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-type-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Value 'XXXX' is not a valid format for: [type] in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-type.yaml b/src/test/resources/plans/schema_registry/invalid-type.yaml new file mode 100644 index 00000000..f4ff1467 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-type.yaml @@ -0,0 +1,10 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: XXXX + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" diff --git a/src/test/resources/plans/schema_registry/invalid-unrecognized-property-output.txt b/src/test/resources/plans/schema_registry/invalid-unrecognized-property-output.txt new file mode 100644 index 00000000..5dceccd2 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-unrecognized-property-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Unrecognized field: [what] in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-unrecognized-property.yaml b/src/test/resources/plans/schema_registry/invalid-unrecognized-property.yaml new file mode 100644 index 00000000..d945d85a --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-unrecognized-property.yaml @@ -0,0 +1,12 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + what: "" diff --git a/src/test/resources/plans/schema_registry/no-changes-include-unchanged-plan.json b/src/test/resources/plans/schema_registry/no-changes-include-unchanged-plan.json new file mode 100644 index 00000000..1bd12923 --- /dev/null +++ b/src/test/resources/plans/schema_registry/no-changes-include-unchanged-plan.json @@ -0,0 +1,39 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-1-json", + "action" : "NO_CHANGE", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "schema-2-avro", + "action" : "NO_CHANGE", + "schemaDetails" : { + "type" : "AVRO", + "schema" : "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "schema-3-protobuf", + "action" : "NO_CHANGE", + "schemaDetails" : { + "type" : "PROTOBUF", + "schema" : "syntax = \"proto3\";\npackage com.acme;\n\nmessage OtherRecord {\n int32 an_id = 1;\n}\n", + "file" : null, + "compatibility" : "FULL", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/no-changes-output.txt b/src/test/resources/plans/schema_registry/no-changes-output.txt new file mode 100644 index 00000000..9b86d9f0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/no-changes-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[SUCCESS] There are no necessary changes; the actual state matches the desired state. diff --git a/src/test/resources/plans/schema_registry/no-changes-plan.json b/src/test/resources/plans/schema_registry/no-changes-plan.json new file mode 100644 index 00000000..6dbd87bc --- /dev/null +++ b/src/test/resources/plans/schema_registry/no-changes-plan.json @@ -0,0 +1,5 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/no-changes.yaml b/src/test/resources/plans/schema_registry/no-changes.yaml new file mode 100644 index 00000000..d6f1571d --- /dev/null +++ b/src/test/resources/plans/schema_registry/no-changes.yaml @@ -0,0 +1,25 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\", \"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" + schema-3-protobuf: + type: PROTOBUF + compatibility: FULL + schema: 'syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +} +' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt new file mode 100644 index 00000000..20e85895 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt @@ -0,0 +1,29 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value2 + + type: JSON + + compatibility: FULL + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] json-value3 + + type: JSON + + compatibility: NONE + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 2 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-default-plan.json b/src/test/resources/plans/schema_registry/schema-registry-default-plan.json new file mode 100644 index 00000000..7f302c6a --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-default-plan.json @@ -0,0 +1,28 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value2", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "FULL", + "references" : [ ] + } + }, + { + "name" : "json-value3", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-default.yaml b/src/test/resources/plans/schema_registry/schema-registry-default.yaml new file mode 100644 index 00000000..649428bb --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-default.yaml @@ -0,0 +1,17 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + schemas: + defaults: + compatibility: FULL + +schemas: + json-value2: + type: JSON + file: schema-registry-schema1.json + json-value3: + type: JSON + compatibility: NONE + file: schema-registry-schema1.json \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt new file mode 100644 index 00000000..ae1d2741 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt @@ -0,0 +1,42 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: FULL + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] avro-value1 + + type: AVRO + + compatibility: NONE + + schema: +---------------------- +{"type":"record","name":"TestRecord","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello","type":"string"}]} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] avro-value2 + + type: AVRO + + compatibility: FORWARD + + schema: +---------------------- +{"type":"record","name":"TestRecord2","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello2","type":"string"}]} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 3 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-mix-plan.json b/src/test/resources/plans/schema_registry/schema-registry-mix-plan.json new file mode 100644 index 00000000..65424f72 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-mix-plan.json @@ -0,0 +1,39 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "FULL", + "references" : [ ] + } + }, + { + "name" : "avro-value1", + "action" : "ADD", + "schemaDetails" : { + "type" : "AVRO", + "schema" : "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + }, + { + "name" : "avro-value2", + "action" : "ADD", + "schemaDetails" : { + "type" : "AVRO", + "schema" : "{\"type\":\"record\",\"name\":\"TestRecord2\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello2\",\"type\":\"string\"}]}", + "file" : null, + "compatibility" : "FORWARD", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-mix.yaml b/src/test/resources/plans/schema_registry/schema-registry-mix.yaml new file mode 100644 index 00000000..ac0a4af4 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-mix.yaml @@ -0,0 +1,31 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + schemas: + defaults: + compatibility: FULL + +schemas: + json-value: + type: JSON + file: schema-registry-schema1.json + avro-value1: + type: AVRO + compatibility: NONE + file: schema-registry-schema2.avsc + avro-value2: + type: AVRO + compatibility: FORWARD + schema : '{ + "type": "record", + "name": "TestRecord2", + "namespace": "com.devshawn.kafka.gitops", + "fields": [ + { + "name": "hello2", + "type": "string" + } + ] +}' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt new file mode 100644 index 00000000..5758d549 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt @@ -0,0 +1,16 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] avro-value + + type: AVRO + + compatibility: FULL + + schema: +---------------------- +{"type":"record","name":"TestRecord","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello","type":"string"}]} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-avro-plan.json b/src/test/resources/plans/schema_registry/schema-registry-new-avro-plan.json new file mode 100644 index 00000000..0bfecfc2 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-avro-plan.json @@ -0,0 +1,17 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "avro-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "AVRO", + "schema" : "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + "file" : null, + "compatibility" : "FULL", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-avro.yaml b/src/test/resources/plans/schema_registry/schema-registry-new-avro.yaml new file mode 100644 index 00000000..b7170db7 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-avro.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + avro-value: + type: AVRO + compatibility: FULL + file: schema-registry-schema2.avsc \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt new file mode 100644 index 00000000..1d6b009a --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt @@ -0,0 +1,16 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: BACKWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-json-plan.json b/src/test/resources/plans/schema_registry/schema-registry-new-json-plan.json new file mode 100644 index 00000000..b6e7cd6c --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-json-plan.json @@ -0,0 +1,17 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-json.yaml b/src/test/resources/plans/schema_registry/schema-registry-new-json.yaml new file mode 100644 index 00000000..606cb35a --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-json.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + json-value: + type: JSON + compatibility: BACKWARD + file: schema-registry-schema1.json \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt new file mode 100644 index 00000000..f12da016 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt @@ -0,0 +1,22 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] proto-value + + type: PROTOBUF + + compatibility: NONE + + schema: +---------------------- +syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +} + +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-proto-plan.json b/src/test/resources/plans/schema_registry/schema-registry-new-proto-plan.json new file mode 100644 index 00000000..d2bef497 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-proto-plan.json @@ -0,0 +1,17 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "proto-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "PROTOBUF", + "schema" : "syntax = \"proto3\";\npackage com.acme;\n\nmessage OtherRecord {\n int32 an_id = 1;\n}\n", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-proto.yaml b/src/test/resources/plans/schema_registry/schema-registry-new-proto.yaml new file mode 100644 index 00000000..81f151c8 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-proto.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + proto-value: + type: PROTOBUF + compatibility: NONE + file: schema-registry-schema3.proto \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema1.json b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema1.json new file mode 100644 index 00000000..7e6cfcc0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema1.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "properties": { + "f1": { + "type": "string" + } + } +} diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema2.avsc b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema2.avsc new file mode 100644 index 00000000..8530806d --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema2.avsc @@ -0,0 +1,11 @@ +{ + "type": "record", + "name": "TestRecord", + "namespace": "com.devshawn.kafka.gitops", + "fields": [ + { + "name": "hello", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema3.proto b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema3.proto new file mode 100644 index 00000000..70c11b4b --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema3.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema4.avsc b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema4.avsc new file mode 100644 index 00000000..4cde42fb --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema4.avsc @@ -0,0 +1,51 @@ +{ + "namespace": "dto", + "type": "record", + "name": "CreateOpsDTO", + "fields": [ + { + "name": "externalIds", + "type": { + "type": "array", + "items": "string" + } + }, + { + "name": "attachmentId", + "type": "string" + }, + { + "name": "deliveryOfferId", + "type": "string" + }, + { + "name": "deliveryOfferCode", + "type": "string" + }, + { + "name": "deliveryOfferName", + "type": "string" + }, + { + "name": "inherited", + "type": "boolean" + }, + { + "name": "effectDate", + "type": "int", + "logicalType": "date" + }, + { + "name": "endEffectDate", + "type": "string" + }, + { + "name": "motifEndEffectDate", + "type": "string" + }, + { + "name": "updatedBy", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema5.avsc b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema5.avsc new file mode 100644 index 00000000..204be0af --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema5.avsc @@ -0,0 +1,56 @@ +{ + "namespace": "dto", + "type": "record", + "name": "CreateOpsDTO", + "fields": [ + { + "name": "externalIds", + "type": { + "type": "array", + "items": "string" + } + }, + { + "name": "attachmentId", + "type": "string" + }, + { + "name": "deliveryOfferId", + "type": "string" + }, + { + "name": "deliveryOfferCode", + "type": "string" + }, + { + "name": "deliveryOfferName", + "type": "string" + }, + { + "name": "inherited", + "type": "boolean" + }, + { + "name": "effectDate", + "type": { + "type": "int", + "logicalType": "date" + } + }, + { + "name": "endEffectDate", + "type": { + "type": "int", + "logicalType": "date" + } + }, + { + "name": "motifEndEffectDate", + "type": "string" + }, + { + "name": "updatedBy", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt new file mode 100644 index 00000000..f7959038 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt @@ -0,0 +1,34 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] schema-2-json + + type: JSON + + compatibility: FORWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"$ref":"otherschema"}},"additionalProperties":false} +---------------------- + + references: + + name: otherschema + + subject: schema-1-json + + version: 1 + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-2-avro + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-3-protobuf + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 2 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-plan.json b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-plan.json new file mode 100644 index 00000000..0e66113d --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-plan.json @@ -0,0 +1,33 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-2-json", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "FORWARD", + "references" : [ + { + "name" : "otherschema", + "subject" : "schema-1-json", + "version" : 1 + } + ] + } + }, + { + "name" : "schema-2-avro", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-add-with-reference.yaml b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference.yaml new file mode 100644 index 00000000..4ae0be57 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference.yaml @@ -0,0 +1,19 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-json: + type: JSON + compatibility: FORWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}}, \"additionalProperties\": false}" + references: + - name: otherschema + subject: schema-1-json + version: 1 \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-delete-reference-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-delete-reference-apply-output.txt new file mode 100644 index 00000000..35c5b332 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-delete-reference-apply-output.txt @@ -0,0 +1,59 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] schema-2-json + + type: JSON + + compatibility: FORWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"$ref":"otherschema"}},"additionalProperties":false} +---------------------- + + references: + + name: otherschema + + subject: schema-1-json + + version: 1 + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-2-avro + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-10-json + + +Applied deferred... + +Applying: [DELETE] + +- [SCHEMA] schema-3-protobuf + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-11-json + + +Successfully applied. + +Applying deffered actions: + +Applying: [DELETE] + +- [SCHEMA] schema-10-json + + +Successfully applied. + +Deferred actions successfully applied + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 4 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-delete-reference-plan.json b/src/test/resources/plans/schema_registry/seed-schema-delete-reference-plan.json new file mode 100644 index 00000000..e018d715 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-delete-reference-plan.json @@ -0,0 +1,43 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-2-json", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "FORWARD", + "references" : [ + { + "name" : "otherschema", + "subject" : "schema-1-json", + "version" : 1 + } + ] + } + }, + { + "name" : "schema-2-avro", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-10-json", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-11-json", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} diff --git a/src/test/resources/plans/schema_registry/seed-schema-delete-reference.yaml b/src/test/resources/plans/schema_registry/seed-schema-delete-reference.yaml new file mode 100644 index 00000000..4ae0be57 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-delete-reference.yaml @@ -0,0 +1,19 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-json: + type: JSON + compatibility: FORWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}}, \"additionalProperties\": false}" + references: + - name: otherschema + subject: schema-1-json + version: 1 \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt new file mode 100644 index 00000000..78dfe118 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt @@ -0,0 +1,23 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: FORWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [UPDATE] + +~ [SCHEMA] schema-1-json + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 1 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-2-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-2-plan.json new file mode 100644 index 00000000..5aab8acf --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-2-plan.json @@ -0,0 +1,28 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "FORWARD", + "references" : [ ] + } + }, + { + "name" : "schema-1-json", + "action" : "UPDATE", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml new file mode 100644 index 00000000..f77acb03 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml @@ -0,0 +1,29 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + json-value: + type: JSON + compatibility: FORWARD + schema: '{"type": "object","properties": {"f1": {"type": "string"}}}' + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" + schema-3-protobuf: + type: PROTOBUF + compatibility: FULL + schema: 'syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +} +' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-3-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-3-apply-output.txt new file mode 100644 index 00000000..e6024998 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-3-apply-output.txt @@ -0,0 +1,17 @@ +Executing apply... + +Applying: [UPDATE] + +~ [SCHEMA] schema-1-json + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-3-protobuf + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 0 created, 1 updated, 1 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-3-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-3-plan.json new file mode 100644 index 00000000..484c3167 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-3-plan.json @@ -0,0 +1,22 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-1-json", + "action" : "UPDATE", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"},\"f2\":{\"type\":\"string\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-3.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-3.yaml new file mode 100644 index 00000000..85ee3fc0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-3.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"},\"f2\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt new file mode 100644 index 00000000..a2d681b8 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt @@ -0,0 +1,20 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] schema-2-json + + type: JSON + + compatibility: BACKWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"$ref":"otherschema"}},"additionalProperties":false} +---------------------- + + references: + + name: otherschema + + subject: schema-1-json + + version: -1 + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-4-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-4-plan.json new file mode 100644 index 00000000..29534810 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-4-plan.json @@ -0,0 +1,23 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-2-json", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ + { + "name" : "otherschema", + "subject" : "schema-1-json", + "version" : -1 + } + ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml new file mode 100644 index 00000000..170b767b --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml @@ -0,0 +1,33 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-2-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}}, \"additionalProperties\": false}" + references: + - name: otherschema + subject: schema-1-json + version: -1 + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" + schema-3-protobuf: + type: PROTOBUF + compatibility: FULL + schema: 'syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +} +' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt new file mode 100644 index 00000000..f165628f --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt @@ -0,0 +1,50 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: BACKWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] json-value1 + + type: JSON + + compatibility: NONE + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-2-avro + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-1-json + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-3-protobuf + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 2 created, 0 updated, 3 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist-plan.json new file mode 100644 index 00000000..124b4e9c --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist-plan.json @@ -0,0 +1,38 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "json-value1", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + }, + { + "name" : "schema-1-json", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist.yaml new file mode 100644 index 00000000..3f0cf281 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist.yaml @@ -0,0 +1,19 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + schemas: + blacklist: + prefixed: + - schema-2 + +schemas: + json-value: + type: JSON + compatibility: BACKWARD + schema: '{"type": "object","properties": {"f1": {"type": "string"}}}' + json-value1: + type: JSON + compatibility: NONE + file: schema-registry-schema1.json \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt new file mode 100644 index 00000000..ccd085c1 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt @@ -0,0 +1,29 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: BACKWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] json-value1 + + type: JSON + + compatibility: NONE + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 2 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-plan.json new file mode 100644 index 00000000..14202234 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-plan.json @@ -0,0 +1,17 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-1-json", + "action" : "UPDATE", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"},\"f2\":{\"type\":\"string\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete.yaml new file mode 100644 index 00000000..85ee3fc0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"},\"f2\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-plan.json new file mode 100644 index 00000000..ca6ab83e --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-plan.json @@ -0,0 +1,43 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "json-value1", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + }, + { + "name" : "schema-2-avro", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-1-json", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification.yaml new file mode 100644 index 00000000..c59b92a4 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + json-value: + type: JSON + compatibility: BACKWARD + schema: '{"type": "object","properties": {"f1": {"type": "string"}}}' + json-value1: + type: JSON + compatibility: NONE + file: schema-registry-schema1.json \ No newline at end of file diff --git a/src/test/resources/plans/seed-acl-exists-plan.json b/src/test/resources/plans/seed-acl-exists-plan.json index a0fb8fb2..6a1fd21a 100644 --- a/src/test/resources/plans/seed-acl-exists-plan.json +++ b/src/test/resources/plans/seed-acl-exists-plan.json @@ -1,57 +1,58 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": null, - "previousValue": "compact", - "action": "REMOVE" + "key" : "cleanup.policy", + "value" : null, + "previousValue" : "compact", + "action" : "REMOVE" }, { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-1", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-acl-exists.yaml b/src/test/resources/plans/seed-acl-exists.yaml index 81a5c7f6..1afa3411 100644 --- a/src/test/resources/plans/seed-acl-exists.yaml +++ b/src/test/resources/plans/seed-acl-exists.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-basic-include-unchanged-plan.json b/src/test/resources/plans/seed-basic-include-unchanged-plan.json index a9672db2..b5ae5978 100644 --- a/src/test/resources/plans/seed-basic-include-unchanged-plan.json +++ b/src/test/resources/plans/seed-basic-include-unchanged-plan.json @@ -1,142 +1,143 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "delete-topic", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "delete-topic", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "NO_CHANGE" }, { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "400000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "400000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "topic-with-configs-2", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-2", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "NO_CHANGE" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "test-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "test-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "NO_CHANGE" + "action" : "NO_CHANGE" }, { - "name": "test-service-1", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "new-service-0", - "aclDetails": { - "name": "delete-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:new", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "new-service-0", + "aclDetails" : { + "name" : "delete-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:new", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "new-service-1", - "aclDetails": { - "name": "new-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:new", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "new-service-1", + "aclDetails" : { + "name" : "new-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:new", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-basic-plan.json b/src/test/resources/plans/seed-basic-plan.json index 2d8450cd..32c6f86e 100644 --- a/src/test/resources/plans/seed-basic-plan.json +++ b/src/test/resources/plans/seed-basic-plan.json @@ -1,83 +1,84 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "400000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "400000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "test-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "test-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-1", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "new-service-0", - "aclDetails": { - "name": "delete-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:new", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "new-service-0", + "aclDetails" : { + "name" : "delete-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:new", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "new-service-1", - "aclDetails": { - "name": "new-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:new", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "new-service-1", + "aclDetails" : { + "name" : "new-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:new", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-basic.yaml b/src/test/resources/plans/seed-basic.yaml index d3c3a09b..715a55bc 100644 --- a/src/test/resources/plans/seed-basic.yaml +++ b/src/test/resources/plans/seed-basic.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-blacklist-topics-plan.json b/src/test/resources/plans/seed-blacklist-topics-plan.json index a51fdfa2..b3bc8767 100644 --- a/src/test/resources/plans/seed-blacklist-topics-plan.json +++ b/src/test/resources/plans/seed-blacklist-topics-plan.json @@ -1,50 +1,51 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-2", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-1", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-blacklist-topics.yaml b/src/test/resources/plans/seed-blacklist-topics.yaml index 8b80b3cd..f04cf192 100644 --- a/src/test/resources/plans/seed-blacklist-topics.yaml +++ b/src/test/resources/plans/seed-blacklist-topics.yaml @@ -3,6 +3,7 @@ settings: blacklist: prefixed: - test + - _schemas topics: new-topic: diff --git a/src/test/resources/plans/seed-topic-add-partitions-plan.json b/src/test/resources/plans/seed-topic-add-partitions-plan.json index 9cee3b65..0184e6a6 100644 --- a/src/test/resources/plans/seed-topic-add-partitions-plan.json +++ b/src/test/resources/plans/seed-topic-add-partitions-plan.json @@ -1,45 +1,46 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": 4, - "previousPartitions": 3, - "partitionsAction": "UPDATE", - "replication": null, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : 4, + "previousPartitions" : 3, + "partitionsAction" : "UPDATE", + "replication" : null, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": 10, - "previousPartitions": 6, - "partitionsAction": "UPDATE", - "replication": null, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-2", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : 10, + "previousPartitions" : 6, + "partitionsAction" : "UPDATE", + "replication" : null, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-add-partitions.yaml b/src/test/resources/plans/seed-topic-add-partitions.yaml index 7d93f433..099b56f7 100644 --- a/src/test/resources/plans/seed-topic-add-partitions.yaml +++ b/src/test/resources/plans/seed-topic-add-partitions.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-add-replicas-plan.json b/src/test/resources/plans/seed-topic-add-replicas-plan.json index b194f9fd..99318506 100644 --- a/src/test/resources/plans/seed-topic-add-replicas-plan.json +++ b/src/test/resources/plans/seed-topic-add-replicas-plan.json @@ -1,45 +1,46 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": null, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 3, - "previousReplication": 2, - "replicationAction": "UPDATE" + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : null, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 3, + "previousReplication" : 2, + "replicationAction" : "UPDATE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": null, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 3, - "previousReplication": 2, - "replicationAction": "UPDATE" + "name" : "topic-with-configs-2", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : null, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 3, + "previousReplication" : 2, + "replicationAction" : "UPDATE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-add-replicas.json b/src/test/resources/plans/seed-topic-add-replicas.json index 668f8387..6a4eb537 100644 --- a/src/test/resources/plans/seed-topic-add-replicas.json +++ b/src/test/resources/plans/seed-topic-add-replicas.json @@ -1,83 +1,84 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "segment.bytes", - "value": null, - "previousValue": null, - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : null, + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "topic-with-configs-2", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-2", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "100000", - "previousValue": "60000", - "action": "UPDATE" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : "60000", + "action" : "UPDATE" } ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-add-replicas.yaml b/src/test/resources/plans/seed-topic-add-replicas.yaml index 9a6666ba..4d702729 100644 --- a/src/test/resources/plans/seed-topic-add-replicas.yaml +++ b/src/test/resources/plans/seed-topic-add-replicas.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-modification-2-plan.json b/src/test/resources/plans/seed-topic-modification-2-plan.json index 2064a26b..088321e6 100644 --- a/src/test/resources/plans/seed-topic-modification-2-plan.json +++ b/src/test/resources/plans/seed-topic-modification-2-plan.json @@ -1,82 +1,83 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": null, - "previousValue": "compact", - "action": "REMOVE" + "key" : "cleanup.policy", + "value" : null, + "previousValue" : "compact", + "action" : "REMOVE" }, { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-2", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-modification-2.yaml b/src/test/resources/plans/seed-topic-modification-2.yaml index de2f4f37..d0fd7637 100644 --- a/src/test/resources/plans/seed-topic-modification-2.yaml +++ b/src/test/resources/plans/seed-topic-modification-2.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-modification-3-plan.json b/src/test/resources/plans/seed-topic-modification-3-plan.json index 07ad15e0..129cb086 100644 --- a/src/test/resources/plans/seed-topic-modification-3-plan.json +++ b/src/test/resources/plans/seed-topic-modification-3-plan.json @@ -1,83 +1,84 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "topic-with-configs-2", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-2", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "100000", - "previousValue": "60000", - "action": "UPDATE" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : "60000", + "action" : "UPDATE" } ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-modification-3.yaml b/src/test/resources/plans/seed-topic-modification-3.yaml index 322eca5b..dd5dda4c 100644 --- a/src/test/resources/plans/seed-topic-modification-3.yaml +++ b/src/test/resources/plans/seed-topic-modification-3.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-modification-no-delete-plan.json b/src/test/resources/plans/seed-topic-modification-no-delete-plan.json index 66ba7b10..3bf44322 100644 --- a/src/test/resources/plans/seed-topic-modification-no-delete-plan.json +++ b/src/test/resources/plans/seed-topic-modification-no-delete-plan.json @@ -1,31 +1,32 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-modification-no-delete.yaml b/src/test/resources/plans/seed-topic-modification-no-delete.yaml index 1e62856f..dfddd461 100644 --- a/src/test/resources/plans/seed-topic-modification-no-delete.yaml +++ b/src/test/resources/plans/seed-topic-modification-no-delete.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-modification-plan.json b/src/test/resources/plans/seed-topic-modification-plan.json index e9175241..853e900d 100644 --- a/src/test/resources/plans/seed-topic-modification-plan.json +++ b/src/test/resources/plans/seed-topic-modification-plan.json @@ -1,63 +1,64 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-2", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-1", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-modification.yaml b/src/test/resources/plans/seed-topic-modification.yaml index 1e62856f..dfddd461 100644 --- a/src/test/resources/plans/seed-topic-modification.yaml +++ b/src/test/resources/plans/seed-topic-modification.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-remove-replicas-plan.json b/src/test/resources/plans/seed-topic-remove-replicas-plan.json index 808cd811..29bcbb32 100644 --- a/src/test/resources/plans/seed-topic-remove-replicas-plan.json +++ b/src/test/resources/plans/seed-topic-remove-replicas-plan.json @@ -27,6 +27,7 @@ "topicConfigPlans": [] } ], + "schemaPlans" : [ ], "aclPlans": [ { "name": "Unnamed ACL", diff --git a/src/test/resources/plans/seed-topic-remove-replicas.yaml b/src/test/resources/plans/seed-topic-remove-replicas.yaml index 8b472de7..f8f19a3f 100644 --- a/src/test/resources/plans/seed-topic-remove-replicas.yaml +++ b/src/test/resources/plans/seed-topic-remove-replicas.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/simple-plan.json b/src/test/resources/plans/simple-plan.json index e841e1a1..03744be7 100644 --- a/src/test/resources/plans/simple-plan.json +++ b/src/test/resources/plans/simple-plan.json @@ -1,18 +1,19 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/simple-users-plan.json b/src/test/resources/plans/simple-users-plan.json index f1702e15..60105020 100644 --- a/src/test/resources/plans/simple-users-plan.json +++ b/src/test/resources/plans/simple-users-plan.json @@ -1,123 +1,124 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-user-0", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "test-user-0", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-1", - "aclDetails": { - "name": "*", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "test-user-1", + "aclDetails" : { + "name" : "*", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-2", - "aclDetails": { - "name": "*", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "test-user-2", + "aclDetails" : { + "name" : "*", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-3", - "aclDetails": { - "name": "*", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-user-3", + "aclDetails" : { + "name" : "*", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-4", - "aclDetails": { - "name": "*", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "test-user-4", + "aclDetails" : { + "name" : "*", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-5", - "aclDetails": { - "name": "*", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-user-5", + "aclDetails" : { + "name" : "*", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-6", - "aclDetails": { - "name": "*", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-user-6", + "aclDetails" : { + "name" : "*", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-7", - "aclDetails": { - "name": "*", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-user-7", + "aclDetails" : { + "name" : "*", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/simple-users.yaml b/src/test/resources/plans/simple-users.yaml index 3b5eafd0..6a15658d 100644 --- a/src/test/resources/plans/simple-users.yaml +++ b/src/test/resources/plans/simple-users.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 6 diff --git a/src/test/resources/plans/simple.yaml b/src/test/resources/plans/simple.yaml index a4e985b0..fd59d299 100644 --- a/src/test/resources/plans/simple.yaml +++ b/src/test/resources/plans/simple.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 6 diff --git a/src/test/resources/plans/skip-acls-apply-plan.json b/src/test/resources/plans/skip-acls-apply-plan.json index 7a48cd63..7612c355 100644 --- a/src/test/resources/plans/skip-acls-apply-plan.json +++ b/src/test/resources/plans/skip-acls-apply-plan.json @@ -1,110 +1,111 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "MY_TOPIC", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 1, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "MY_TOPIC", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 1, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "another.topic.0", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 1, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "another.topic.0", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 1, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "ADD" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "ADD" }, { - "key": "segment.bytes", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "segment.bytes", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "another.topic.0", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "another.topic.0", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "MY_TOPIC", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "MY_TOPIC", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "my-other-service-0", - "aclDetails": { - "name": "another.topic.0", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "my-other-service-0", + "aclDetails" : { + "name" : "another.topic.0", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "my-other-service-1", - "aclDetails": { - "name": "my-other-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "my-other-service-1", + "aclDetails" : { + "name" : "my-other-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } diff --git a/src/test/resources/plans/skip-acls-plan.json b/src/test/resources/plans/skip-acls-plan.json index c70d8bc9..acc5ef62 100644 --- a/src/test/resources/plans/skip-acls-plan.json +++ b/src/test/resources/plans/skip-acls-plan.json @@ -1,44 +1,45 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "MY_TOPIC", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 1, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "MY_TOPIC", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 1, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "another.topic.0", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 1, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "another.topic.0", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 1, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "ADD" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "ADD" }, { - "key": "segment.bytes", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "segment.bytes", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] } ], - "aclPlans": [] -} + "schemaPlans" : [ ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/skip-acls.yaml b/src/test/resources/plans/skip-acls.yaml index d52c0a41..6335ac75 100644 --- a/src/test/resources/plans/skip-acls.yaml +++ b/src/test/resources/plans/skip-acls.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: MY_TOPIC: partitions: 6 diff --git a/src/test/resources/plans/topics-and-services-plan.json b/src/test/resources/plans/topics-and-services-plan.json index ebccf613..265fb98b 100644 --- a/src/test/resources/plans/topics-and-services-plan.json +++ b/src/test/resources/plans/topics-and-services-plan.json @@ -1,110 +1,111 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "MY_TOPIC", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "MY_TOPIC", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "another.topic.0", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "another.topic.0", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "ADD" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "ADD" }, { - "key": "segment.bytes", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "segment.bytes", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "another.topic.0", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "another.topic.0", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "MY_TOPIC", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "MY_TOPIC", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "my-other-service-0", - "aclDetails": { - "name": "another.topic.0", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "my-other-service-0", + "aclDetails" : { + "name" : "another.topic.0", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "my-other-service-1", - "aclDetails": { - "name": "my-other-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "my-other-service-1", + "aclDetails" : { + "name" : "my-other-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/topics-and-services.yaml b/src/test/resources/plans/topics-and-services.yaml index db878157..2ba4950f 100644 --- a/src/test/resources/plans/topics-and-services.yaml +++ b/src/test/resources/plans/topics-and-services.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: MY_TOPIC: partitions: 6