From 832fa2173c1f46dda7cfab0ec837efc8ccd94736 Mon Sep 17 00:00:00 2001 From: 893751156 <893751156@qq.com> Date: Sun, 11 Mar 2018 16:06:26 +0800 Subject: [PATCH 1/8] Necessary statement for loadavg --- bindings.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/bindings.c b/bindings.c index fe521a33..aff625c8 100644 --- a/bindings.c +++ b/bindings.c @@ -66,6 +66,7 @@ enum { LXC_TYPE_PROC_STAT, LXC_TYPE_PROC_DISKSTATS, LXC_TYPE_PROC_SWAPS, + LXC_TYPE_PROC_LOADAVG, }; struct file_info { @@ -4144,7 +4145,8 @@ int proc_getattr(const char *path, struct stat *sb) strcmp(path, "/proc/uptime") == 0 || strcmp(path, "/proc/stat") == 0 || strcmp(path, "/proc/diskstats") == 0 || - strcmp(path, "/proc/swaps") == 0) { + strcmp(path, "/proc/swaps") == 0 || + strcmp(path, "/proc/loadavg") == 0) { sb->st_size = 0; sb->st_mode = S_IFREG | 00444; sb->st_nlink = 1; @@ -4158,13 +4160,14 @@ int proc_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offs struct fuse_file_info *fi) { if (filler(buf, ".", NULL, 0) != 0 || - filler(buf, "..", NULL, 0) != 0 || - filler(buf, "cpuinfo", NULL, 0) != 0 || - filler(buf, "meminfo", NULL, 0) != 0 || - filler(buf, "stat", NULL, 0) != 0 || - filler(buf, "uptime", NULL, 0) != 0 || - filler(buf, "diskstats", NULL, 0) != 0 || - filler(buf, "swaps", NULL, 0) != 0) + filler(buf, "..", NULL, 0) != 0 || + filler(buf, "cpuinfo", NULL, 0) != 0 || + filler(buf, "meminfo", NULL, 0) != 0 || + filler(buf, "stat", NULL, 0) != 0 || + filler(buf, "uptime", NULL, 0) != 0 || + filler(buf, "diskstats", NULL, 0) != 0 || + filler(buf, "swaps", NULL, 0) != 0 || + filler(buf, "loadavg", NULL, 0) != 0 ) return -EINVAL; return 0; } @@ -4186,6 +4189,8 @@ int proc_open(const char *path, struct fuse_file_info *fi) type = LXC_TYPE_PROC_DISKSTATS; else if (strcmp(path, "/proc/swaps") == 0) type = LXC_TYPE_PROC_SWAPS; + else if(strcmp(path,"/proc/loadavg") == 0) + type = LXC_TYPE_PROC_LOADAVG; if (type == -1) return -ENOENT; @@ -4243,6 +4248,8 @@ int proc_read(const char *path, char *buf, size_t size, off_t offset, return proc_diskstats_read(buf, size, offset, fi); case LXC_TYPE_PROC_SWAPS: return proc_swaps_read(buf, size, offset, fi); + case LXC_TYPE_PROC_LOADAVG: + return proc_loadavg_read(buf, size, offset, fi); default: return -EINVAL; } From 49e316c8bfcb96bafd8c05463fefc30c93d29f03 Mon Sep 17 00:00:00 2001 From: 893751156 <893751156@qq.com> Date: Sun, 11 Mar 2018 16:12:24 +0800 Subject: [PATCH 2/8] load_daemon --- bindings.c | 283 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) diff --git a/bindings.c b/bindings.c index aff625c8..89697469 100644 --- a/bindings.c +++ b/bindings.c @@ -80,6 +80,80 @@ struct file_info { int cached; }; +/* The function of hash table.*/ +#define LOAD_SIZE 100 /*the size of hash_table */ +#define FLUSH_TIME 5 /*the flush rate */ +#define DEPTH_DIR 3 /*the depth of per cgroup */ +/* The function of calculate loadavg .*/ +#define FSHIFT 11 /* nr of bits of precision */ +#define FIXED_1 (1<> FSHIFT) +#define LOAD_FRAC(x) LOAD_INT(((x) & (FIXED_1-1)) * 100) +struct load_node { + char *cg; /*cg */ + unsigned long avenrun[3]; /* Load averages */ + unsigned int run_pid; + unsigned int total_pid; + unsigned int last_pid; + int cfd; /* the file descriptor of the mounted cgroup */ + struct load_node *next; + struct load_node **pre; +}; +struct load_head { + /* The lock is about insert load_node and refresh load_node.To the first load_node of each hash bucket, insert and + refresh in this hash bucket is mutually exclusive. */ + pthread_mutex_t lock; + /* The rdlock is about read loadavg and delete load_node.To each hash bucket, read and delete is + mutually exclusive. But at the same time, we allow paratactic read operation. This rdlock is at list level.*/ + pthread_rwlock_t rdlock; + /* The rilock is about read loadavg and insert load_node.To the first load_node of each hash bucket, read and insert is + mutually exclusive. But at the same time, we allow paratactic read operation.*/ + pthread_rwlock_t rilock; + struct load_node *next; +}; +static struct load_head *load_hash[LOAD_SIZE]; /* hash table */ +void init_load(void) +{ + int i; + + for (i = 0; i < LOAD_SIZE; i++) { + load_hash[i] = (struct load_head *)malloc(sizeof(struct load_head)); + load_hash[i]->next = NULL; + if (pthread_mutex_init(&load_hash[i]->lock, NULL) != 0) + goto out; + if (pthread_rwlock_init(&load_hash[i]->rdlock, NULL) != 0) + goto out; + if (pthread_rwlock_init(&load_hash[i]->rilock, NULL) != 0) + goto out; + } + return; +out: + free(load_hash[i]); + lxcfs_error("%s\n", "lock init error"); + exit(1); +} + +static struct load_node *del_node(struct load_node *n, int locate) +{ + struct load_node *g; + + pthread_rwlock_wrlock(&load_hash[locate]->rdlock); + if (n->next == NULL) { + *(n->pre) = NULL; + } else { + *(n->pre) = n->next; + n->next->pre = n->pre; + } + g = n->next; + printf("delete node %s \n", n->cg); + free(n->cg); + free(n); + pthread_rwlock_unlock(&load_hash[locate]->rdlock); + return g; +} /* Reserve buffer size to account for file size changes. */ #define BUF_RESERVE_SIZE 512 @@ -4109,6 +4183,215 @@ static int proc_swaps_read(char *buf, size_t size, off_t offset, return rv; } +/* +Find the process pid from cgroup path. +eg:from /sys/fs/cgroup/cpu/docker/containerid/cgroup.procs to find the process pid. + pid_buf : put pid to buffer. + dpath : the path of cgroup. eg: /docker/containerid or /docker/containerid/child-cgroup ... + depth : the depth of cgroup in container. + sum : return the number of pid . + cfd : the file descriptor of the mounted cgroup. eg: /sys/fs/cgroup/cpu +*/ +static int calc_pid(char ***pid_buf, char *dpath, int depth, int sum, int cfd) +{ + DIR *dir; + int fd; + struct dirent *file; + /* path = dpath + "/cgroup.procs" + /0 */ + char *path = (char *)malloc(strlen(dpath) + 20); + + strcpy(path, dpath); + fd = openat(cfd, path, O_RDONLY); + if (fd < 0) + return sum; + + if ((dir = fdopendir(fd)) == NULL) { + return sum; + } else { + while (((file = readdir(dir)) != NULL) && depth > 0) { + if (strncmp(file->d_name, ".", 1) == 0) + continue; + else + if (file->d_type == DT_DIR) { + /* path + '/' + d_name +/0 */ + char *path_dir = (char *)malloc(strlen(path)+2+sizeof(file->d_name)); + strcpy(path_dir, path); + strcat(path_dir, "/"); + strcat(path_dir, file->d_name); + int pd = depth - 1; + sum = calc_pid(pid_buf, path_dir, pd, sum, cfd); + free(path_dir); + } + } + closedir(dir); + } + + close(fd); + if (sum == -1) { + free(path); + return -1; + } + strcat(path, "/cgroup.procs"); + FILE *f = NULL; + size_t linelen = 0; + char *line = NULL; + + fd = openat(cfd, path, O_RDONLY); + if (fd < 0) + return sum; + + if (!(f = fdopen(fd, "r"))) + return sum; + + while (getline(&line, &linelen, f) != -1) { + char **pid; + pid = (char **)realloc(*pid_buf, sizeof(char *) * (sum + 1)); + if (pid == NULL) { + lxcfs_error("%s\n", "realloc error!"); + for (; sum > 0; sum--) + free((*pid_buf)[sum - 1]); + return -1; + } + *pid_buf = pid; + *(*pid_buf + sum) = (char *)malloc(strlen(line) + 3); + strcpy(*(*pid_buf + sum), line); + sum++; + } + fclose(f); + free(path); + return sum; +} +/* + * a1 = a0 * e + a * (1 - e) + */ +static unsigned long +calc_load(unsigned long load, unsigned long exp, unsigned long active) +{ + unsigned long newload; + active = active > 0 ? active * FIXED_1 : 0; + newload = load * exp + active * (FIXED_1 - exp); + if (active >= load) + newload += FIXED_1 - 1; + + return newload / FIXED_1; +} + +static int refresh_load(struct load_node *p, char *path) +{ + FILE *f = NULL; + char **idbuf; + char proc_path[50]; + int i, run_pid = 0, total_pid = 0, last_pid = 0; + char *line = NULL; + size_t linelen = 0; + idbuf = (char **)malloc(sizeof(char *)); + int sum = calc_pid(&idbuf, path, DEPTH_DIR, 0, p->cfd); + DIR *dp; + struct dirent *file; + /* normal exit */ + if (sum == 0) { + free(idbuf); + return sum; + } + /* abnormal exit : realloc() failed*/ + if (sum == -1) { + free(idbuf); + return 0; + } + for (i = 0; i < sum; i++) { + idbuf[i][strlen(idbuf[i])-1] = '\0'; + sprintf(proc_path, "/proc/%s/task", idbuf[i]); + if (!(dp = opendir(proc_path))) { + lxcfs_error("%s\n", "calc error when opendir."); + continue; + } else + while ((file = readdir(dp)) != NULL) { + if (strncmp(file->d_name, ".", 1) == 0) + continue; + total_pid++; + /* We make the biggest pid become last_pid.*/ + last_pid = (atof(file->d_name) > last_pid) ? atof(file->d_name) : last_pid; + sprintf(proc_path, "/proc/%s/task/%s/status", idbuf[i], file->d_name); + if ((f = fopen(proc_path, "r")) != NULL) { + while (getline(&line, &linelen, f) != -1) { + /* find State */ + if ((line[0] == 'S') && (line[1] == 't')) + break; + } + if ((line[7] == 'R') || (line[7] == 'D')) + run_pid++; + fclose(f); + } + } + closedir(dp); + } + /*Calculate the loadavg.*/ + p->avenrun[0] = calc_load(p->avenrun[0], EXP_1, run_pid); + p->avenrun[1] = calc_load(p->avenrun[1], EXP_5, run_pid); + p->avenrun[2] = calc_load(p->avenrun[2], EXP_15, run_pid); + p->run_pid = run_pid; + p->total_pid = total_pid; + p->last_pid = last_pid; + + for (; i > 0; i--) + free(idbuf[i-1]); + free(idbuf); + free(line); + return sum; +} +/* + Traverse the hash table and update it. +*/ +void *load_begin(void *arg) +{ + + char *path = NULL; + int i, sum; + struct load_node *f; + int first_node; + while (1) { + clock_t time1 = clock(); + for (i = 0; i < LOAD_SIZE; i++) { + pthread_mutex_lock(&load_hash[i]->lock); + if (load_hash[i]->next == NULL) { + pthread_mutex_unlock(&load_hash[i]->lock); + continue; + } + f = load_hash[i]->next; + first_node = 1; + while (f) { + path = (char *)malloc(strlen(f->cg) + 2); + sprintf(path, "%s%s", *(f->cg) == '/' ? "." : "", f->cg); + sum = refresh_load(f, path); + if (sum == 0) + f = del_node(f, i); + else + f = f->next; + free(path); + if (first_node == 1) { + first_node = 0; + pthread_mutex_unlock(&load_hash[i]->lock); + } + } + } + clock_t time2 = clock(); + usleep(FLUSH_TIME * 1000000 - (int)((time2 - time1) * 1000000 / CLOCKS_PER_SEC)); + } +} + +pthread_t load_daemon(void) +{ + init_load(); + int ret; + pthread_t pid; + ret = pthread_create(&pid, NULL, load_begin, NULL); + if (ret != 0) { + lxcfs_error("%s\n", "Create pthread error!\n"); + exit(1); + } + return pid; +} + static off_t get_procfile_size(const char *which) { FILE *f = fopen(which, "r"); From e6ad86d050b9f288ffb4958e11ea9f62018fa3a1 Mon Sep 17 00:00:00 2001 From: 893751156 <893751156@qq.com> Date: Sun, 11 Mar 2018 16:13:09 +0800 Subject: [PATCH 3/8] load_daemon --- bindings.h | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings.h b/bindings.h index c663287b..b66fa2db 100644 --- a/bindings.h +++ b/bindings.h @@ -32,5 +32,6 @@ extern int proc_open(const char *path, struct fuse_file_info *fi); extern int proc_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi); extern int proc_access(const char *path, int mask); +extern pthread_t load_daemon(void); #endif /* __LXCFS_BINDINGS_H */ From d3f5c8f662415abe65746d6ff322a57aa927a070 Mon Sep 17 00:00:00 2001 From: 893751156 <893751156@qq.com> Date: Sun, 11 Mar 2018 16:14:49 +0800 Subject: [PATCH 4/8] load_daemon --- lxcfs.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lxcfs.c b/lxcfs.c index 62cfd350..bd54295f 100644 --- a/lxcfs.c +++ b/lxcfs.c @@ -848,6 +848,8 @@ int main(int argc, char *argv[]) char *pidfile = NULL, *v = NULL; size_t pidfile_len; bool debug = false; + pthread_t pid; + char *error; /* * what we pass to fuse_main is: * argv[0] -s [-f|-d] -o allow_other,directio argv[1] NULL @@ -901,7 +903,16 @@ int main(int argc, char *argv[]) } if ((pidfd = set_pidfile(pidfile)) < 0) goto out; + dlerror(); /* Clear any existing error */ + pthread_t (*load_daemon)(void); + load_daemon = (pthread_t (*)(void)) dlsym(dlopen_handle, "load_daemon"); + error = dlerror(); + if (error != NULL) { + lxcfs_error("%s\n", error); + return -1; + } + pid = load_daemon(); if (!fuse_main(nargs, newargv, &lxcfs_ops, NULL)) ret = EXIT_SUCCESS; From 639008159149be928d93065cea808c016fd2b172 Mon Sep 17 00:00:00 2001 From: 893751156 <893751156@qq.com> Date: Sun, 11 Mar 2018 16:16:00 +0800 Subject: [PATCH 5/8] load_free --- lxcfs.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lxcfs.c b/lxcfs.c index bd54295f..2126278d 100644 --- a/lxcfs.c +++ b/lxcfs.c @@ -915,7 +915,20 @@ int main(int argc, char *argv[]) pid = load_daemon(); if (!fuse_main(nargs, newargv, &lxcfs_ops, NULL)) ret = EXIT_SUCCESS; - + if (pthread_cancel(pid) == 0) { + sleep(5); + dlerror(); /* Clear any existing error */ + void (*load_free)(void); + + load_free = (void (*)(void)) dlsym(dlopen_handle, "load_free"); + error = dlerror(); + if (error != NULL) { + lxcfs_error("%s\n", error); + return -1; + } + load_free(); + } else + lxcfs_error("%s\n", "load_free error!"); out: if (dlopen_handle) dlclose(dlopen_handle); From f2e197391471f242813fd8fc35eb0b1925cffe9a Mon Sep 17 00:00:00 2001 From: 893751156 <893751156@qq.com> Date: Sun, 11 Mar 2018 16:16:31 +0800 Subject: [PATCH 6/8] load_free --- bindings.h | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings.h b/bindings.h index b66fa2db..a37abf65 100644 --- a/bindings.h +++ b/bindings.h @@ -33,5 +33,6 @@ extern int proc_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi); extern int proc_access(const char *path, int mask); extern pthread_t load_daemon(void); +extern void load_free(void); #endif /* __LXCFS_BINDINGS_H */ From fa3b2966ebd216e004832e372ab8889c5098f518 Mon Sep 17 00:00:00 2001 From: 893751156 <893751156@qq.com> Date: Sun, 11 Mar 2018 16:17:40 +0800 Subject: [PATCH 7/8] load_free --- bindings.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/bindings.c b/bindings.c index 89697469..a6280681 100644 --- a/bindings.c +++ b/bindings.c @@ -154,6 +154,41 @@ static struct load_node *del_node(struct load_node *n, int locate) pthread_rwlock_unlock(&load_hash[locate]->rdlock); return g; } + +void load_free(void) +{ + int i; + struct load_node *f, *p; + + for (i = 0; i < LOAD_SIZE; i++) { + pthread_mutex_lock(&load_hash[i]->lock); + pthread_rwlock_wrlock(&load_hash[i]->rilock); + pthread_rwlock_wrlock(&load_hash[i]->rdlock); + if (load_hash[i]->next == NULL) { + pthread_mutex_unlock(&load_hash[i]->lock); + pthread_mutex_destroy(&load_hash[i]->lock); + pthread_rwlock_unlock(&load_hash[i]->rilock); + pthread_rwlock_destroy(&load_hash[i]->rilock); + pthread_rwlock_unlock(&load_hash[i]->rdlock); + pthread_rwlock_destroy(&load_hash[i]->rdlock); + free(load_hash[i]); + continue; + } + for (f = load_hash[i]->next; f; ) { + free(f->cg); + p = f->next; + free(f); + f = p; + } + pthread_mutex_unlock(&load_hash[i]->lock); + pthread_mutex_destroy(&load_hash[i]->lock); + pthread_rwlock_unlock(&load_hash[i]->rilock); + pthread_rwlock_destroy(&load_hash[i]->rilock); + pthread_rwlock_unlock(&load_hash[i]->rdlock); + pthread_rwlock_destroy(&load_hash[i]->rdlock); + free(load_hash[i]); + } +} /* Reserve buffer size to account for file size changes. */ #define BUF_RESERVE_SIZE 512 From 3eb7d3ebbc29ddb7e582af851cd214aa5717f3c9 Mon Sep 17 00:00:00 2001 From: 893751156 <893751156@qq.com> Date: Sun, 11 Mar 2018 16:20:59 +0800 Subject: [PATCH 8/8] loadavg --- bindings.c | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/bindings.c b/bindings.c index a6280681..974d40f9 100644 --- a/bindings.c +++ b/bindings.c @@ -92,6 +92,23 @@ struct file_info { #define EXP_15 2037 /* 1/exp(5sec/15min) */ #define LOAD_INT(x) ((x) >> FSHIFT) #define LOAD_FRAC(x) LOAD_INT(((x) & (FIXED_1-1)) * 100) + +static int calc_hash(char *name) +{ + unsigned int hash = 0; + unsigned int x = 0; + + while (*name) { + hash = (hash << 4) + *name++; + x = hash & 0xf0000000; + if (x != 0) { + hash ^= (x >> 24); + hash &= ~x; + } + } + return ((hash & 0x7fffffff) % LOAD_SIZE); +} + struct load_node { char *cg; /*cg */ unsigned long avenrun[3]; /* Load averages */ @@ -136,6 +153,39 @@ void init_load(void) exit(1); } +static void insert_node(struct load_node **n, int locate) +{ + pthread_mutex_lock(&load_hash[locate]->lock); + pthread_rwlock_wrlock(&load_hash[locate]->rilock); + struct load_node *f = load_hash[locate]->next; + load_hash[locate]->next = *n; + + (*n)->pre = &(load_hash[locate]->next); + if (f) + f->pre = &((*n)->next); + (*n)->next = f; + pthread_mutex_unlock(&load_hash[locate]->lock); + pthread_rwlock_unlock(&load_hash[locate]->rilock); +} +/* Find spectial node. It means find if it doesn't return NULL. */ +static struct load_node *locate_node(char *cg, int locate) +{ + struct load_node *f = NULL; + int i = 0; + + pthread_rwlock_rdlock(&load_hash[locate]->rilock); + pthread_rwlock_rdlock(&load_hash[locate]->rdlock); + if (load_hash[locate]->next == NULL) { + pthread_rwlock_unlock(&load_hash[locate]->rilock); + return f; + } + f = load_hash[locate]->next; + pthread_rwlock_unlock(&load_hash[locate]->rilock); + while (f && ((i = strcmp(f->cg, cg)) != 0)) + f = f->next; + return f; +} + static struct load_node *del_node(struct load_node *n, int locate) { struct load_node *g; @@ -4414,6 +4464,84 @@ void *load_begin(void *arg) } } +static int proc_loadavg_read(char *buf, size_t size, off_t offset, + struct fuse_file_info *fi) +{ + struct fuse_context *fc = fuse_get_context(); + struct file_info *d = (struct file_info *)fi->fh; + char *cg; + size_t total_len = 0; + + char *cache = d->buf; + + struct load_node *n; + int hash; + int cfd; + unsigned long a, b, c; + + if (offset) { + if (offset > d->size) + return -EINVAL; + if (!d->cached) + return 0; + int left = d->size - offset; + total_len = left > size ? size : left; + memcpy(buf, cache + offset, total_len); + return total_len; + } + + pid_t initpid = lookup_initpid_in_store(fc->pid); + if (initpid <= 0) + initpid = fc->pid; + cg = get_pid_cgroup(initpid, "cpu"); + if (!cg) + return read_file("/proc/loadavg", buf, size, d); + + prune_init_slice(cg); + hash = calc_hash(cg); + n = locate_node(cg, hash); + + /* first time */ + if (n == NULL) { + if (!find_mounted_controller("cpu", &cfd)) { + pthread_rwlock_unlock(&load_hash[hash]->rdlock); + return 0; + } + + n = (struct load_node *)malloc(sizeof(struct load_node)); + n->cg = (char *)malloc(strlen(cg)+1); + strcpy(n->cg, cg); + n->avenrun[0] = 0; + n->avenrun[1] = 0; + n->avenrun[2] = 0; + n->run_pid = 0; + n->total_pid = 1; + n->last_pid = initpid; + n->cfd = cfd; + insert_node(&n, hash); + } + a = n->avenrun[0] + (FIXED_1/200); + b = n->avenrun[1] + (FIXED_1/200); + c = n->avenrun[2] + (FIXED_1/200); + total_len = snprintf(d->buf, d->buflen, "%lu.%02lu %lu.%02lu %lu.%02lu %d/%d %d\n", + LOAD_INT(a), LOAD_FRAC(a), + LOAD_INT(b), LOAD_FRAC(b), + LOAD_INT(c), LOAD_FRAC(c), + n->run_pid, n->total_pid, n->last_pid); + pthread_rwlock_unlock(&load_hash[hash]->rdlock); + if (total_len < 0 || total_len >= d->buflen) { + lxcfs_error("%s\n", "failed to write to cache"); + return 0; + } + d->size = (int)total_len; + d->cached = 1; + + if (total_len > size) + total_len = size; + memcpy(buf, d->buf, total_len); + return total_len; +} + pthread_t load_daemon(void) { init_load();