Merge branch 'OpenVisualCloud:main' into main

.github/workflows/codeql.yml

@@ -81,12 +81,9 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y git gcc meson python3 python3-pip pkg-config libnuma-dev libjson-c-dev libpcap-dev libgtest-dev libsdl2-dev libsdl2-ttf-dev libssl-dev
-        sudo apt-get install -y cmake build-essential autoconf libtool pkg-config
+        sudo apt-get install -y cmake build-essential autoconf libtool pkg-config python3-pyelftools ninja-build
         sudo apt-get install -y dpdk-dev
 
-    - name: Install the python package
-      run: pip install pyelftools ninja
-
     - name: Install IMTL
       run: |
         git clone --depth 1 https://github.com/OpenVisualCloud/Media-Transport-Library.git imtl

.github/workflows/dependency-review.yml

@@ -24,4 +24,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@7bbfa034e752445ea40215fff1c3bf9597993d3f # v3.1.3
+        uses: actions/dependency-review-action@01bc87099ba56df1e897b6874784491ea6309bc4 # v3.1.4

.github/workflows/ubuntu-build.yml

@@ -38,12 +38,9 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y git gcc meson python3 python3-pip pkg-config libnuma-dev libjson-c-dev libpcap-dev libgtest-dev libsdl2-dev libsdl2-ttf-dev libssl-dev
-        sudo apt-get install -y cmake build-essential autoconf libtool pkg-config
+        sudo apt-get install -y cmake build-essential autoconf libtool pkg-config python3-pyelftools ninja-build
         sudo apt-get install -y dpdk-dev
 
-    - name: Install the python packages
-      run: pip install pyelftools ninja
-
     - name: Install IMTL
       run: |
         git clone --depth 1 https://github.com/OpenVisualCloud/Media-Transport-Library.git imtl

README.md

@@ -109,10 +109,16 @@ Usage: media_proxy [OPTION]
 - The Authentication function of the Media Proxy interfaces is currently missing. This feature is still under development, and the current implementation is weak in defending against network attacks.
 
 ## Support
-If you have any problem during use this project, you could find support in following ways.
-- Search in the project [documents](https://github.com/OpenVisualCloud/Media-Communications-Mesh/tree/main/docs)
-- Ask your question in the [Discussions](https://github.com/OpenVisualCloud/Media-Communications-Mesh/discussions/categories/q-a)
-- Submit [Issues](https://github.com/OpenVisualCloud/Media-Communications-Mesh/issues) for bug.
+
+If you encounter any issues or need assistance, there are several ways to seek support:
+
+- **Project Documents:** Search for solutions in the [project documents](https://github.com/OpenVisualCloud/Media-Communications-Mesh/tree/main/docs).
+- **Discussions:** Ask questions and seek help in the [Discussions](https://github.com/OpenVisualCloud/Media-Communications-Mesh/discussions/categories/q-a) section on the project's GitHub page.
+- **Issue Submission:** Report bugs and specific issues by submitting them on the [Issues](https://github.com/OpenVisualCloud/Media-Communications-Mesh/issues) page of the project's GitHub repository.
+
+Before submitting an issue, please check the existing documentation and discussions to avoid duplication and streamline the support process.
+
+We are here to help, so don't hesitate to reach out if you need assistance.
 
 ## Note
 This project is under development.

deployment/DaemonSet/media-proxy.yaml

@@ -1,3 +1,20 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mcm
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: memif-socket-fs
+  namespace: mcm
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 64Mi
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -6,11 +23,16 @@ metadata:
   labels:
     app: media-proxy
 spec:
+  selector:
+    matchLabels:
+      app: media-proxy
   template:
     metadata:
       labels:
         app: media-proxy
     spec:
+      nodeSelector:
+        node-role.mcm.intel.com/worker: "true"
       containers:
         - name: media-proxy
           image: mcm/media-proxy:latest
@@ -61,14 +83,3 @@ spec:
           emptyDir:
             medium: Memory
             sizeLimit: 4Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: memif-socket-fs
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 64Mi

deployment/setup.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+echo "Create MCM cluster."
+minikube start --nodes 3 --namespace mcm --insecure-registry "10.0.0.0/24"
+
+echo "Enable minikube addons."
+minikube addons enable registry
+minikube addons enable metrics-server
+
+echo "Set name for worker nodes."
+worker_node_0="minikube-m02"
+worker_node_1="minikube-m03"
+
+echo "Load MCM docker image into cluster."
+minikube image load mcm/media-proxy:latest
+
+echo "Set node label to MCM worker nodes."
+kubectl label nodes ${worker_node_0} node-role.mcm.intel.com/worker=true
+kubectl label nodes ${worker_node_1} node-role.mcm.intel.com/worker=true
+
+echo "Deploy Media Proxy DaemonSet."
+kubectl create -f DaemonSet/media-proxy.yaml

media-proxy/Dockerfile

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-FROM ubuntu:22.04@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f as builder
+FROM ubuntu:22.04@sha256:8eab65df33a6de2844c9aefd19efe8ddb87b7df5e9185a4ab73af936225685bb as builder
 LABEL maintainer="[email protected]"
 
 ENV DEBIAN_FRONTEND noninteractive
@@ -44,8 +44,8 @@ RUN mkdir -p /opt/mcm/3rd_party/
 RUN adduser ${USER} sudo \
         && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
-# Install latest versions of CMake and Conan using pip3 package installer
-RUN python3 -m pip install --no-cache-dir "conan<2"
+# Install Conan using pip3 package installer.
+RUN python3 -m pip install --no-cache-dir conan=1.62.0
 
 # Build DPDK/IMTL
 ENV DPDK_VER=23.03
@@ -84,7 +84,7 @@ WORKDIR $MCM_DIR
 RUN ./build.sh
 
 ## Re-build container for optimised runtime environment using clean Ubuntu release
-FROM ubuntu:22.04@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f
+FROM ubuntu:22.04@sha256:8eab65df33a6de2844c9aefd19efe8ddb87b7df5e9185a4ab73af936225685bb
 
 ARG USER=docker
 RUN useradd -l -m -s /bin/bash ${USER}

media-proxy/README.md

@@ -55,13 +55,50 @@ docker run --privileged -v /dev/vfio:/dev/vfio mcm/media-proxy:latest
 The "--privileged" argument is necessary to access NIC hardware with DPDK driver.
 
 ### Kubernetes
-The Media Proxy designed to run as DaemonSet of MCM system, to save the system resouce.
-There's a sample YAML file integrated in this repo as reference.
+The Media Proxy is designed to operate as a DaemonSet within the MCM system, aiming to conserve system resources. For details, a sample YAML file has been integrated into this repository as a reference.
+
+To deploy the Media Proxy as a DaemonSet in your Kubernetes cluster, you can use the following command:
+
+### Dependencies Installation
+
+Before deploying the Media Proxy on Kubernetes using Minikube, you'll need to ensure that Docker & Minikube are installed. Follow these steps to install all dependencies:
+
+1. **Install Docker**: You can find installation instructions for Docker based on your operating system from [Docker's official documentation](https://docs.docker.com/get-docker/).
+
+2. **Install a Hypervisor**: Minikube requires a virtualization solution to create a virtual machine. You can use [VirtualBox](https://www.virtualbox.org/) or [KVM](https://www.linux-kvm.org/page/Main_Page) as the hypervisor.
+
+3. **Install kubectl**: kubectl is a command line tool for interacting with the Kubernetes API server. You can find installation instructions for kubectl [here](https://kubernetes.io/docs/tasks/tools/install-kubectl/).
+
+4. **Install Minikube**: Follow the installation instructions for Minikube based on your operating system from [Minikube's official documentation](https://minikube.sigs.k8s.io/docs/start/).
+
+Once you've completed the above steps, you'll have Docker, a hypervisor, kubectl, and Minikube installed and ready to deploy the Media Proxy on your local Kubernetes cluster.
+
+#### Setup K8s Cluster
+Before deploy media proxy to the K8s cluster, you need to execute following steps to setup the K8s cluster to be ready for MCM.
+
+1. Start the K8s Cluster, and add MCM worker node on it.
 
 ```bash
+$ minikube start
+$ minikube node add -n 1
+```
+
+2. Set the label for the worker node.
+
+```bash
+$ kubectl label nodes minikube-m02 mcm.intel.com/role=worker
+```
+
+#### Deploy Media Proxy
+
+```bash
+$ cd Media-Communications-Mesh
 $ kubectl apply -f deployment/DaemonSet/media-proxy.yaml
+$ kubectl get daemonsets.apps -n mcm
 ```
 
+If all commands are executed successfully, you will see the MCM media proxy deployed as a K8s DaemonSet to the MCM worker node (labeled with "mcm.intel.com/role=worker").
+
 ## Known Issues
 - There is one bug with default docker.io package installation (version 20.10.25-0ubuntu1~22.04.2) with Ubuntu 22.04.3 LTS. The [`USER` command](https://github.com/moby/moby/issues/46355) and [`chown` command](https://github.com/moby/moby/issues/46161) don't work as expected. It's preferred to install docker-ce package following [instruction from docker community](https://docs.docker.com/engine/install/ubuntu/).
 

media-proxy/src/tcp_controller.cc

@@ -298,7 +298,6 @@ void RunTCPServer(ProxyContext* ctx)
         return;
     }
 
-    // INFO("ready and listening");
     INFO("TCP Server listening on %s", ctx->getTCPListenAddress().c_str());
 
     // signal(SIGINT, intHandler);

sdk/include/mcm_dp.h

@@ -51,8 +51,8 @@ typedef struct {
     struct {
         uint16_t seq_num; /* Sequence number */
         uint32_t timestamp; /* Timestamp */
-    } metadata;
-    size_t len;
+    } metadata; /**< filled by sender side */
+    size_t len; /**< size of data filled in "data" */
     void* data;
 } mcm_buffer;
 

sdk/samples/sender_app.c

@@ -94,8 +94,9 @@ int read_test_data(FILE* fp, mcm_buffer* buf, uint32_t width, uint32_t height, v
         perror("Error when read frame file");
         ret = -1;
     }
-    //TODO: metadata is not carried to receiver side
+
     buf->metadata.seq_num = buf->metadata.timestamp = frm_idx++;
+    buf->len = frame_size;
 
     return ret;
 }

sdk/src/impl_memif.c

@@ -247,6 +247,8 @@ mcm_conn_context* mcm_create_connection_memif(mcm_conn_param* svc_args, memif_co
     }
     if (ret != MEMIF_ERR_SUCCESS) {
         log_info("memif_create: %s", memif_strerror(ret));
+        free(shm_conn);
+        shm_conn = NULL;
         memif_delete_socket(&memif_socket);
         return NULL;
     }
@@ -255,13 +257,19 @@ mcm_conn_context* mcm_create_connection_memif(mcm_conn_param* svc_args, memif_co
         ret = memif_poll_event(shm_conn->sockfd, -1);
         if (ret != MEMIF_ERR_SUCCESS) {
             log_error("Create memif connection failed.");
+            free(shm_conn);
+            shm_conn = NULL;
+            memif_delete_socket(&memif_socket);
             return NULL;
         }
     } while (shm_conn->is_connected == 0);
 
     conn_ctx = calloc(1, sizeof(mcm_conn_context));
     if (conn_ctx == NULL) {
         log_error("Outof Memory.");
+        free(shm_conn);
+        shm_conn = NULL;
+        memif_delete_socket(&memif_socket);
         return NULL;
     }
 
@@ -415,6 +423,10 @@ int memif_enqueue_buffer(mcm_conn_context* conn_ctx, mcm_buffer* buf)
             return -1;
         }
 
+        *(uint16_t *)(memif_conn->working_bufs[0].data) = buf->metadata.seq_num;
+        *(uint32_t *)(memif_conn->working_bufs[0].data + sizeof(buf->metadata.seq_num)) = buf->metadata.timestamp;
+        *(size_t *)(memif_conn->working_bufs[0].data + sizeof(buf->metadata.seq_num) + sizeof(buf->metadata.timestamp)) = buf->len;
+
         err = memif_tx_burst(memif_conn->conn, memif_conn->qid, &memif_conn->working_bufs[0], 1, &buf_num);
         if (err != MEMIF_ERR_SUCCESS) {
             log_error("memif_tx_burst: %s", memif_strerror(err));

sdk/src/mcm_dp.c

@@ -13,7 +13,7 @@
 #include "mcm_dp.h"
 #include "media_proxy_ctrl.h"
 
-static void parse_memIF_param(mcm_conn_param* request, memif_socket_args_t* memif_socket_args, memif_conn_args_t* memif_conn_args)
+static void parse_memif_param(mcm_conn_param* request, memif_socket_args_t* memif_socket_args, memif_conn_args_t* memif_conn_args)
 {
     char type_str[8] = "";
     char interface_name[32];
@@ -142,7 +142,7 @@ mcm_conn_context* mcm_create_connection(mcm_conn_param* param)
         break;
     case PROTO_MEMIF:;
         memif_conn_param memif_param = {};
-        parse_memIF_param(param, &(memif_param.socket_args), &(memif_param.conn_args));
+        parse_memif_param(param, &(memif_param.socket_args), &(memif_param.conn_args));
         /* Connect memif connection. */
         conn_ctx = mcm_create_connection_memif(param, &memif_param);
         conn_ctx->session_id = 0;