docker-compose-node-gpu.yaml

services:
  dcgm_exporter:
    image: nvcr.io/nvidia/k8s/dcgm-exporter:${DCGM_EXPORTER_IMAGE_TAG}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [ gpu ]
    restart: unless-stopped
    environment:
      - DCGM_EXPORTER_NO_HOSTNAME=1
    cap_add:
      - SYS_ADMIN
    ports:
      - "${DCGM_EXPORTER_HOST_PORT}:9400"
    networks:
      - gpu_metrics

#  nvidia-smi-exporter:
#    privileged: true
#    build: ./nvidia-smi-exporter
#    runtime: nvidia
#    pid: "host"
#    volumes:
#      - prometheus_textfiles:/run/prometheus
#      - /var/run/docker.sock:/var/run/docker.sock
#    restart: unless-stopped

  node_exporter:
    image: prom/node-exporter:latest
    restart: unless-stopped
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--web.listen-address=:9401'
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    ports:
      - "9401:9401"
    networks:
      - gpu_metrics

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.47.1
    privileged: true
    devices:
      - /dev/kmsg:/dev/kmsg
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker:/var/lib/docker:ro
      #- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
    restart: unless-stopped
    command:
      - '-port=8098'
    ports:
      - "8098:8098"
    networks:
      - gpu_metrics

volumes:
  prometheus_data:
    driver: local
  grafana_data:
    driver: local
  prometheus_textfiles:
    driver_opts:
      type: tmpfs
      device: tmpfs

networks:
  gpu_metrics:
    driver: bridge