Optimize api using webflux and kotlin coroutines

.github/workflows/cd.yml

@@ -0,0 +1,61 @@
+name: main
+
+on:
+  push:
+    branches: [ main ]
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/rinha-backend
+  test:
+    runs-on: ubuntu-20.04
+    needs:
+      - build
+    permissions:
+      contents: write
+    steps:
+      - run: docker-compose up -d
+
+      - name: Load test
+        run: |
+          git clone --single-branch --quiet https://github.com/zanfranceschi/rinha-de-backend-2023-q3
+          cd rinha-de-backend-2023-q3
+          wget -q https://repo1.maven.org/maven2/io/gatling/highcharts/gatling-charts-highcharts-bundle/3.9.5/gatling-charts-highcharts-bundle-3.9.5-bundle.zip
+          unzip gatling-charts-highcharts-bundle-3.9.5-bundle.zip
+          cd gatling-charts-highcharts-bundle-3.9.5
+          ./bin/gatling.sh -rm local -s RinhaBackendSimulation -rd "DESCRICAO" -rf $WORKSPACE/user-files/results -sf $WORKSPACE/user-files/simulations -rsf $WORKSPACE/user-files/resources
+          echo GATLING_OUTPUT_FOLDER=$(ls $WORKSPACE/user-files/results | sort | head -n 1) >> $GITHUB_ENV
+        env:
+          WORKSPACE: ${{ github.workspace }}/rinha-de-backend-2023-q3/stress-test
+      - run: echo ${{ env.GATLING_OUTPUT_FOLDER }}
+      - run: |
+          # replace string DESCRICAO with the commit sha in file index.html inside GATLING OUTPUT FOLDER
+          sed -i "s/DESCRICAO/${{ github.sha }}/g" ${{ github.workspace }}/rinha-de-backend-2023-q3/stress-test/user-files/results/${{ env.GATLING_OUTPUT_FOLDER }}/index.html
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ${{ github.workspace }}/rinha-de-backend-2023-q3/stress-test/user-files/results/${{ env.GATLING_OUTPUT_FOLDER }}
+          destination_dir: ${{ env.GATLING_OUTPUT_FOLDER }}
+      - run: echo "GH_REPO=$(echo ${{ github.repository }} | cut -d "/" -f 2)" >> $GITHUB_ENV
+      - run: echo "[Deploying to https://${{ github.repository_owner }}.github.io/${{ env.GH_REPO }}/${{ env.GATLING_OUTPUT_FOLDER }}](https://${{ github.repository_owner }}.github.io/${{ env.GH_REPO }}/${{ env.GATLING_OUTPUT_FOLDER }})" >> "${GITHUB_STEP_SUMMARY}"
+
+

.github/workflows/ci.yml

@@ -1,8 +1,6 @@
 name: main
 
 on:
-  push:
-    branches: [ main ]
   pull_request:
     branches: [ main ]
 
@@ -14,31 +12,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./Dockerfile
-          push: true
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/rinha-backend
-
       - run: |
+          docker build . -t fzanutto/rinha-backend
           docker-compose up -d
 
       - name: Load test
         run: |
           git clone --single-branch --quiet https://github.com/zanfranceschi/rinha-de-backend-2023-q3
           cd rinha-de-backend-2023-q3
-          wget https://repo1.maven.org/maven2/io/gatling/highcharts/gatling-charts-highcharts-bundle/3.9.5/gatling-charts-highcharts-bundle-3.9.5-bundle.zip
+          wget -q https://repo1.maven.org/maven2/io/gatling/highcharts/gatling-charts-highcharts-bundle/3.9.5/gatling-charts-highcharts-bundle-3.9.5-bundle.zip
           unzip gatling-charts-highcharts-bundle-3.9.5-bundle.zip
           cd gatling-charts-highcharts-bundle-3.9.5
           ./bin/gatling.sh -rm local -s RinhaBackendSimulation -rd "DESCRICAO" -rf $WORKSPACE/user-files/results -sf $WORKSPACE/user-files/simulations -rsf $WORKSPACE/user-files/resources

Dockerfile

@@ -6,4 +6,4 @@ RUN gradle assemble
 FROM openjdk:17
 COPY --from=BUILD /temp/build/libs/rinha-backend-1.1.jar app.jar
 
-ENTRYPOINT ["java","-jar", "-noverify", "/app.jar"]
+ENTRYPOINT ["java","-jar", "/app.jar"]

build.gradle.kts

@@ -5,13 +5,6 @@ plugins {
 	id("io.spring.dependency-management") version "1.1.2"
 	kotlin("jvm") version "1.8.22"
 	kotlin("plugin.spring") version "1.8.22"
-	kotlin("plugin.allopen") version "1.8.0"
-}
-
-allOpen {
-	annotation("jakarta.persistence.Entity")
-	annotation("jakarta.persistence.Embeddable")
-	annotation("jakarta.persistence.MappedSuperclass")
 }
 
 group = "com.fzanutto"
@@ -34,14 +27,17 @@ repositories {
 dependencies {
 	implementation("org.springframework.boot:spring-boot-starter-validation")
 	implementation("org.springframework.boot:spring-boot-starter-cache")
-	implementation("org.springframework.boot:spring-boot-starter-web")
-	implementation("org.springframework.boot:spring-boot-starter-data-jpa")
+	implementation("org.springframework.boot:spring-boot-starter-webflux")
 	implementation("org.springframework.boot:spring-boot-starter-data-redis")
-	implementation("org.postgresql:postgresql")
-	implementation("org.flywaydb:flyway-core")
+	implementation("org.springframework.boot:spring-boot-starter-data-r2dbc")
+	implementation("io.r2dbc:r2dbc-postgresql:0.8.13.RELEASE")
+	runtimeOnly("org.postgresql:postgresql")
 	implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
 	implementation("org.jetbrains.kotlin:kotlin-reflect")
 	annotationProcessor("org.springframework.boot:spring-boot-configuration-processor")
+
+	implementation("io.projectreactor.kotlin:reactor-kotlin-extensions")
+	implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor")
 }
 
 tasks.withType<KotlinCompile> {

docker-compose-local.yml

@@ -4,6 +4,8 @@ services:
     image: postgres:latest
     command: postgres -c 'max_connections=600'
     user: postgres
+    volumes:
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
     restart: always
     healthcheck:
       test: [ "CMD-SHELL", "pg_isready" ]

docker-compose.yml

@@ -1,11 +1,11 @@
 version: "3.8"
 services:
-  api1:
+  api1: &api1
     image: fzanutto/rinha-backend
     hostname: api1
     environment:
       - SERVER_PORT=8000
-      - DATABASE_URL=jdbc:postgresql://postgres:5432/rinha?currentSchema=public
+      - DATABASE_URL=r2dbc:postgresql://postgres:5432/rinha?currentSchema=public
       - DATABASE_USERNAME=postgres
       - DATABASE_PASSWORD=root
       - REDIS_HOST=redis
@@ -17,28 +17,11 @@ services:
     deploy:
       resources:
         limits:
-          cpus: "0.3"
-          memory: "0.7GB"
+          cpus: "0.4"
+          memory: "0.75GB"
 
   api2:
-    image: fzanutto/rinha-backend
-    hostname: api2
-    environment:
-      - SERVER_PORT=8000
-      - DATABASE_URL=jdbc:postgresql://postgres:5432/rinha?currentSchema=public
-      - DATABASE_USERNAME=postgres
-      - DATABASE_PASSWORD=root
-      - REDIS_HOST=redis
-    depends_on:
-      - postgres
-      - redis
-    expose:
-      - "8000"
-    deploy:
-      resources:
-        limits:
-          cpus: "0.3"
-          memory: "0.7GB"
+    <<: *api1
 
   nginx:
     image: nginx:latest
@@ -52,13 +35,15 @@ services:
     deploy:
       resources:
         limits:
-          cpus: "0.1"
+          cpus: "0.15"
           memory: "0.2GB"
 
   postgres:
     image: postgres:latest
-    command: postgres -c 'max_connections=600'
+    command: postgres -c 'max_connections=400'
     user: postgres
+    volumes:
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
     healthcheck:
       test: ["CMD-SHELL", "pg_isready"]
       interval: 3s
@@ -78,12 +63,13 @@ services:
         window: 60s
       resources:
         limits:
-          cpus: "0.7"
-          memory: "0.9GB"
+          cpus: "0.50"
+          memory: "1.0GB"
 
   redis:
     image: redis
     hostname: redis
+    command: ["redis-server", "--appendonly", "no", "--save ''"]
     ports:
       - "6379:6379"
     environment:
@@ -92,5 +78,5 @@ services:
     deploy:
       resources:
         limits:
-          cpus: "0.1"
-          memory: "0.5GB"
+          cpus: "0.05"
+          memory: "0.3GB"

init.sql

@@ -0,0 +1,14 @@
+CREATE EXTENSION IF NOT EXISTS "pg_trgm";
+
+CREATE TABLE IF NOT EXISTS person(
+    id          UUID PRIMARY KEY,
+    nickname    VARCHAR(32) NOT NULL UNIQUE,
+    name        VARCHAR(100) NOT NULL,
+    birthday    DATE         NOT NULL,
+    stack       VARCHAR(1024),
+    search      VARCHAR(1156) GENERATED ALWAYS AS (
+        name || ' ' || nickname || ' ' || stack
+    ) STORED
+);
+
+CREATE INDEX IF NOT EXISTS idx_person_search ON person USING gist (search gist_trgm_ops(siglen = 64));

nginx.conf

@@ -1,8 +1,5 @@
 events {
-    worker_connections 1024;
-    multi_accept on;
-    use epoll;
-    accept_mutex on;
+    worker_connections 2000;
 }
 http {
     upstream api {

src/main/kotlin/com/fzanutto/rinhabackend/CacheConfig.kt

@@ -0,0 +1,18 @@
+package com.fzanutto.rinhabackend
+
+import com.fzanutto.rinhabackend.entity.PersonEntity
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import org.springframework.data.redis.connection.RedisConnectionFactory
+import org.springframework.data.redis.core.RedisTemplate
+
+@Configuration
+class CacheConfig {
+
+    @Bean
+    fun redisTemplate(redisConnectionFactory: RedisConnectionFactory): RedisTemplate<String, PersonEntity> {
+        return RedisTemplate<String, PersonEntity>().apply {
+            connectionFactory = redisConnectionFactory
+        }
+    }
+}

src/main/kotlin/com/fzanutto/rinhabackend/RinhaBackendApplication.kt

@@ -1,13 +1,33 @@
 package com.fzanutto.rinhabackend
 
+import com.fzanutto.rinhabackend.converter.ListToStringWriteConverter
+import com.fzanutto.rinhabackend.converter.StringToListReadConverter
+import io.r2dbc.spi.ConnectionFactory
 import org.springframework.boot.autoconfigure.SpringBootApplication
 import org.springframework.boot.runApplication
 import org.springframework.cache.annotation.EnableCaching
+import org.springframework.context.annotation.Configuration
+import org.springframework.data.r2dbc.config.AbstractR2dbcConfiguration
+import org.springframework.data.r2dbc.repository.config.EnableR2dbcRepositories
 
 @SpringBootApplication
 @EnableCaching
+@EnableR2dbcRepositories
 class RinhaBackendApplication
 
 fun main(args: Array<String>) {
 	runApplication<RinhaBackendApplication>(*args)
 }
+
+@Configuration
+class Config(
+	private val connectionFactory: ConnectionFactory
+): AbstractR2dbcConfiguration() {
+	override fun connectionFactory(): ConnectionFactory {
+		return connectionFactory
+	}
+
+	override fun getCustomConverters(): MutableList<Any> {
+		return mutableListOf(StringToListReadConverter(), ListToStringWriteConverter())
+	}
+}