From 7909eb99d713efa7152015b7b68050aa6565b54d Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Sat, 2 Sep 2023 05:23:48 +0530 Subject: [PATCH] difference rendering --- ...ies.yml => docker-compose-dependencies.yml | 3 - pom.xml | 7 ++ .../ai/diffy/analysis/DynamicAnalyzer.scala | 15 +++- src/main/scala/ai/diffy/flat/FlatObject.scala | 2 +- .../interpreter/http/HttpLambdaServer.java | 3 + .../scala/ai/diffy/lifter/JsonLifter.scala | 2 +- .../proxy/ReactorHttpDifferenceProxy.java | 1 + src/test/resources/application.yml | 34 +++++++ src/test/resources/lambda.js | 89 +++++++++++++++++++ src/test/resources/payload.json | 71 +++++++++++++++ src/test/scala/ai/diffy/IntegrationTest.java | 62 +++++++++++++ .../ai/diffy/compare/DifferenceTest.java | 25 ++++++ 12 files changed, 308 insertions(+), 6 deletions(-) rename docker-compose-dependecies.yml => docker-compose-dependencies.yml (98%) create mode 100644 src/test/resources/application.yml create mode 100644 src/test/resources/lambda.js create mode 100644 src/test/resources/payload.json create mode 100644 src/test/scala/ai/diffy/IntegrationTest.java create mode 100644 src/test/scala/ai/diffy/compare/DifferenceTest.java diff --git a/docker-compose-dependecies.yml b/docker-compose-dependencies.yml similarity index 98% rename from docker-compose-dependecies.yml rename to docker-compose-dependencies.yml index 1075813..cc3ef12 100644 --- a/docker-compose-dependecies.yml +++ b/docker-compose-dependencies.yml @@ -63,7 +63,6 @@ services: - ./etc/promtail-local.yaml:/etc/promtail/promtail-local.yaml - ./data/logs:/app/logs depends_on: - - diffy - loki prometheus: @@ -75,8 +74,6 @@ services: - --config.file=/etc/prometheus.yaml ports: - "9090:9090" - depends_on: - - diffy grafana: image: grafana/grafana:7.4.0-ubuntu diff --git a/pom.xml b/pom.xml index 67e010b..b721a45 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,13 @@ + + org.projectlombok + lombok + 1.18.28 + provided + + org.scala-lang scala-library diff --git a/src/main/scala/ai/diffy/analysis/DynamicAnalyzer.scala b/src/main/scala/ai/diffy/analysis/DynamicAnalyzer.scala index 0731272..23f03b5 100644 --- a/src/main/scala/ai/diffy/analysis/DynamicAnalyzer.scala +++ b/src/main/scala/ai/diffy/analysis/DynamicAnalyzer.scala @@ -3,13 +3,26 @@ package ai.diffy.analysis import ai.diffy.analysis.DynamicAnalyzer.decodeFieldMap import ai.diffy.lifter.{FieldMap, JsonLifter, Message} import ai.diffy.repository.DifferenceResultRepository +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import scala.jdk.CollectionConverters.MapHasAsScala /** * Filters a DifferenceAnalyzer using a specified time range to output another DifferenceAnalyzer */ object DynamicAnalyzer { def decodeFieldMap(payload: String): FieldMap = { - JsonLifter.decode(s"{\"value\":$payload}", classOf[FieldMap]) + objectNodeToFieldMap(JsonLifter.decode(payload).asInstanceOf[ObjectNode]) + } + def objectNodeToFieldMap(objectNode: ObjectNode): FieldMap ={ + val acc = new java.util.HashMap[String, Object]() + objectNode.fields().forEachRemaining(entry => { + acc.put(entry.getKey(), entry.getValue()) + }) + if(acc.containsKey("headers")) { + acc.put("headers", objectNodeToFieldMap(acc.get("headers").asInstanceOf[ObjectNode])) + } + new FieldMap(acc.asScala.toMap) } } class DynamicAnalyzer(repository: DifferenceResultRepository) { diff --git a/src/main/scala/ai/diffy/flat/FlatObject.scala b/src/main/scala/ai/diffy/flat/FlatObject.scala index 1d3441a..3d13bea 100644 --- a/src/main/scala/ai/diffy/flat/FlatObject.scala +++ b/src/main/scala/ai/diffy/flat/FlatObject.scala @@ -63,7 +63,7 @@ object FlatObject { case byteBuffer: ByteBuffer => lift(new String(byteBuffer.asReadOnlyBuffer().array)) case jsonNode: JsonNode => lift(JsonLifter.lift(jsonNode)) case null => FlatNull - case fm: FieldMap => lift(fm.value) + case fm: FieldMap => FlatStruct(lift(fm.value).asInstanceOf[FlatMap[FlatPrimitive[String], FlatObject]]) case _ => FlatStruct(FlatMap(mkMap(a) map {case (k,v) => FlatPrimitive(k) -> lift(v)})) } } diff --git a/src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java b/src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java index 8fafcf2..7c8c6e3 100644 --- a/src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java +++ b/src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java @@ -43,6 +43,9 @@ public HttpLambdaServer(int port, Source httpLambda) { ).bindNow(); } + public void shutdown() { + server.disposeNow(); + } public static void main(String[] args) throws Exception { HttpLambdaServer primary = new HttpLambdaServer(Integer.parseInt(args[0]), new File("src/main/scala/ai/diffy/interpreter/http/master.js")); HttpLambdaServer secondary = new HttpLambdaServer(Integer.parseInt(args[1]), new File("src/main/scala/ai/diffy/interpreter/http/master.js")); diff --git a/src/main/scala/ai/diffy/lifter/JsonLifter.scala b/src/main/scala/ai/diffy/lifter/JsonLifter.scala index 7779509..f422c18 100644 --- a/src/main/scala/ai/diffy/lifter/JsonLifter.scala +++ b/src/main/scala/ai/diffy/lifter/JsonLifter.scala @@ -62,7 +62,7 @@ object JsonLifter { def encode(item: Any): String = Mapper.writer.writeValueAsString(item) def areMapInsteadofObjectKeys(fields: Set[String]): Boolean = - fields.size > 20 || fields.exists{ field => + fields.size > 50 || fields.exists{ field => field.length > 100 || field.matches("[0-9].*") || // starts with a digit !field.matches("[_a-zA-Z0-9]*") // contains non-alphanumeric characters diff --git a/src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java b/src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java index cf2cef5..b901fe3 100644 --- a/src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java +++ b/src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java @@ -6,6 +6,7 @@ import ai.diffy.functional.endpoints.SeptaDependentEndpoint; import ai.diffy.functional.topology.Async; import ai.diffy.functional.topology.ControlFlowLogger; +import ai.diffy.functional.topology.InvocationLogger; import ai.diffy.lifter.AnalysisRequest; import ai.diffy.lifter.HttpLifter; import ai.diffy.repository.DifferenceResultRepository; diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml new file mode 100644 index 0000000..562e00c --- /dev/null +++ b/src/test/resources/application.yml @@ -0,0 +1,34 @@ +spring.application.name: "diffy" +server: + port: ${http.port:8888} + +proxy: + port: 8880 + +candidate: "localhost:9000" + +master: + primary: "localhost:9100" + secondary: "localhost:9200" + +service: + protocol : "http" + +serviceName : "Default Sample Service" + +allowHttpSideEffects: true + +spring: + mongodb: + embedded: + version: "5.0.6" +management: + endpoints: + web: + exposure: + include: health,info,prometheus + +logging: + pattern: + console: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx" + file: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx" \ No newline at end of file diff --git a/src/test/resources/lambda.js b/src/test/resources/lambda.js new file mode 100644 index 0000000..9715b42 --- /dev/null +++ b/src/test/resources/lambda.js @@ -0,0 +1,89 @@ +(request) => { + const { + uri, + method, + path, + params, + headers, + body + } = request; + + const response = { + status: '200 OK', + headers: {}, + body: { + "message": "Success", + "response": { + "id": 98758734857, + "app": 12, + "resource": { + "id": 150, + "app": null, + "label": "Test label", + "hfu": { + "id": 898, + "app": null, + "createdDate": null, + "lastModifiedDate": null, + "name": "test_name", + "uuid": null + }, + "custom": false, + "delhuf": false + }, + "title": "Test title from candidate server", + "abcd": "text", + "related": [ + "ioi" + ], + "data": null, + "flabel": "Test", + "tcon": null, + "type": "UYU", + "gb": [], + "meas": null, + "audy": null, + "stpe": null, + "slim": 0, + "zoomInData": null, + "improvement": null, + "isdterange": true, + "meta": null, + "intcof": null, + "metqu": null, + "fif": { + "id": null, + "requestResource": null, + "resource": null, + "updatable": false, + "empty": true + }, + "modified": false, + "description": null, + "initdat": null, + "applyModifiedData": false, + "sortConfiguration": null, + "pasfor": null, + "originalTitle": "Test original title", + "uuid": "345345346", + "cofn": null, + "rol": null, + "accessible": true, + "hasDel": false, + "witype": null, + "empwi": false, + "syd": null, + "asgr": [], + "ismod": false, + "ughuygegg": false, + "deleted": false, + "newef": false, + "efor": null + }, + "path": null + } + } + + response.body = body; + return response; +} \ No newline at end of file diff --git a/src/test/resources/payload.json b/src/test/resources/payload.json new file mode 100644 index 0000000..eea1798 --- /dev/null +++ b/src/test/resources/payload.json @@ -0,0 +1,71 @@ +{ + "message": "Success", + "response": { + "id": 98758734857, + "app": 12, + "resource": { + "id": 150, + "app": null, + "label": "Test label", + "hfu": { + "id": 898, + "app": null, + "createdDate": null, + "lastModifiedDate": null, + "name": "test_name", + "uuid": null + }, + "custom": false, + "delhuf": false + }, + "title": "Test title from candidate server", + "abcd": "text", + "related": [ + "ioi" + ], + "data": null, + "flabel": "Test", + "tcon": null, + "type": "UYU", + "gb": [], + "meas": null, + "audy": null, + "stpe": null, + "slim": 0, + "zoomInData": null, + "improvement": null, + "isdterange": true, + "meta": null, + "intcof": null, + "metqu": null, + "fif": { + "id": null, + "requestResource": null, + "resource": null, + "updatable": false, + "empty": true + }, + "modified": false, + "description": null, + "initdat": null, + "applyModifiedData": false, + "sortConfiguration": null, + "pasfor": null, + "originalTitle": "Test original title", + "uuid": "345345346", + "cofn": null, + "rol": null, + "accessible": true, + "hasDel": false, + "witype": null, + "empwi": false, + "syd": null, + "asgr": [], + "ismod": false, + "ughuygegg": false, + "deleted": false, + "newef": false, + "efor": null + }, + "path": null +} \ No newline at end of file diff --git a/src/test/scala/ai/diffy/IntegrationTest.java b/src/test/scala/ai/diffy/IntegrationTest.java new file mode 100644 index 0000000..043c837 --- /dev/null +++ b/src/test/scala/ai/diffy/IntegrationTest.java @@ -0,0 +1,62 @@ +package ai.diffy; + +import ai.diffy.interpreter.http.HttpLambdaServer; +import ai.diffy.proxy.ReactorHttpDifferenceProxy; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.io.FileSystemResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.util.FileCopyUtils; +import org.springframework.web.client.RestTemplate; +import static org.junit.jupiter.api.Assertions.*; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; + +@SpringBootTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class IntegrationTest { + RestTemplate restTemplate = new RestTemplate(); + @Autowired + Settings settings; + + @Autowired + ReactorHttpDifferenceProxy proxy; + private Integer extractPort(Downstream downstream) { + HostPort hostPort = (HostPort) downstream; + return hostPort.port(); + } + HttpLambdaServer primary, secondary, candidate; + @BeforeAll + public void setup() throws IOException { + primary = new HttpLambdaServer(extractPort(settings.primary()), new File("src/test/resources/lambda.js")); + secondary = new HttpLambdaServer(extractPort(settings.secondary()), new File("src/test/resources/lambda.js")); + candidate = new HttpLambdaServer(extractPort(settings.candidate()), new File("src/test/resources/lambda.js")); + } + + @AfterAll + public void shutdown() { + primary.shutdown(); + secondary.shutdown(); + candidate.shutdown(); + } + + @Test + public void warmup() throws Exception { + String proxyUrl = "http://localhost:"+settings.servicePort()+"/base"; + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + FileSystemResource payload = new FileSystemResource("src/test/resources/payload.json"); + String json = FileCopyUtils.copyToString(new InputStreamReader(payload.getInputStream())); + String response = restTemplate.postForObject(proxyUrl, new HttpEntity(json, headers), String.class); + assertEquals(json, response); + } +} diff --git a/src/test/scala/ai/diffy/compare/DifferenceTest.java b/src/test/scala/ai/diffy/compare/DifferenceTest.java new file mode 100644 index 0000000..0d276bc --- /dev/null +++ b/src/test/scala/ai/diffy/compare/DifferenceTest.java @@ -0,0 +1,25 @@ +package ai.diffy.compare; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.FileSystemResource; +import org.springframework.util.FileCopyUtils; + +import java.io.IOException; +import java.io.InputStreamReader; + +@Slf4j +public class DifferenceTest { + + @Test + public void payloadTest() throws IOException { + FileSystemResource payload = new FileSystemResource("src/test/resources/payload.json"); + String json = FileCopyUtils.copyToString(new InputStreamReader(payload.getInputStream())); + log.info(json); + Difference diff = Difference.apply(json, json); + diff.flattened().foreach(tuple -> { + log.info("{} - {}", tuple._1(), tuple._2()); + return null; + }); + } +}