Compare commits
8 Commits
da2bdac120
...
9cbb2d6330
Author | SHA1 | Date |
---|---|---|
JT Acquaviva | 9cbb2d6330 | |
JT Acquaviva | fdf70a0e1d | |
JT Acquaviva | 9d46cd6d83 | |
JT Acquaviva | 7312936310 | |
JT Acquaviva | 136cccb67d | |
JT Acquaviva | db7e403a10 | |
JT Acquaviva | fed97e48bc | |
JT Acquaviva | 79e601775e |
|
@ -1,4 +1,5 @@
|
||||||
#include <ngi.h>
|
#include <ngi.h>
|
||||||
|
#include <ngi_log.h>
|
||||||
#include <assert.h>
|
#include <assert.h>
|
||||||
#include <float.h>
|
#include <float.h>
|
||||||
#include <math.h>
|
#include <math.h>
|
||||||
|
@ -23,6 +24,98 @@ int server_reduce(FILE * file, ngi_op_t op, ngi_type_t type, void * out_buff, si
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* this function should be called by IME client on the compute node
|
||||||
|
*/
|
||||||
|
int ngi_client_side_reduce(FILE * file, ngi_op_t op, ngi_type_t type, void * out_buff, size_t off, size_t size){
|
||||||
|
ngi_dbg("Starting offset %lu size %lu Byte\n",off, size);
|
||||||
|
assert(size <= NGI_EXT_SZ);
|
||||||
|
fseek(file, off, SEEK_SET);
|
||||||
|
|
||||||
|
assert(type == NGI_TYPE_FLOAT);
|
||||||
|
assert(op == NGI_OP_MAX);
|
||||||
|
int res = ngi_server_side_reduce(file, op, type, out_buff, off, size);
|
||||||
|
assert(res == 0);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* this function should be called by IME server on the storage appliance
|
||||||
|
*/
|
||||||
|
|
||||||
|
int ngi_server_side_reduce(FILE * file, ngi_op_t op, ngi_type_t type, void * out_buff, size_t off, size_t ext_sz){
|
||||||
|
|
||||||
|
ngi_dbg("Starting offset %lu size %lu Byte\n",off, ext_sz);
|
||||||
|
assert(type == NGI_TYPE_FLOAT);
|
||||||
|
assert(op == NGI_OP_MAX);
|
||||||
|
assert(ext_sz <= NGI_EXT_SZ);
|
||||||
|
|
||||||
|
float * data = malloc(ext_sz);
|
||||||
|
fseek(file, off, 0);
|
||||||
|
fread(data, ext_sz, 1, file);
|
||||||
|
|
||||||
|
float mx = -INFINITY;
|
||||||
|
for(size_t i=0; i < ext_sz/sizeof(float); i++){
|
||||||
|
mx = data[i] > mx ? data[i]: mx;
|
||||||
|
}
|
||||||
|
ngi_dbg("Result: %f \n", mx);
|
||||||
|
*(float*) out_buff = mx;
|
||||||
|
|
||||||
|
free(data);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* this function split the reduction in a set of client server transaction
|
||||||
|
*/
|
||||||
|
int ngi_reduce_stub(FILE * file, ngi_op_t op, ngi_type_t type, void * out_buff, size_t off, size_t size){
|
||||||
|
fseek(file, off, SEEK_SET);
|
||||||
|
|
||||||
|
assert(type == NGI_TYPE_FLOAT);
|
||||||
|
assert(op == NGI_OP_MAX);
|
||||||
|
u_int32_t nb_tx = size / NGI_EXT_SZ;
|
||||||
|
// handle remainder in extend computation
|
||||||
|
if (size % NGI_EXT_SZ != 0)
|
||||||
|
nb_tx += 1;
|
||||||
|
|
||||||
|
float *tmp_red = (float*) malloc(sizeof(float*) * nb_tx);
|
||||||
|
|
||||||
|
/* will be executed in parallel withint the client */
|
||||||
|
ngi_dbg("Reduction from %lu to %lu Byte in %u transactions\n", off, size, nb_tx);
|
||||||
|
for(u_int32_t i = 0; i < nb_tx ; i++){
|
||||||
|
size_t ext_off = off + NGI_EXT_SZ * i;
|
||||||
|
int res = ngi_client_side_reduce(file, op, type, &tmp_red[i], ext_off, NGI_EXT_SZ);
|
||||||
|
assert(res == 0);
|
||||||
|
|
||||||
|
}
|
||||||
|
float mx = -INFINITY;
|
||||||
|
for(u_int32_t i = 0; i < nb_tx ; i++){
|
||||||
|
mx = tmp_red[i] > mx ? tmp_red[i]: mx;
|
||||||
|
}
|
||||||
|
|
||||||
|
*(float*) out_buff = mx;
|
||||||
|
free(tmp_red);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int ngi_reduce(FILE * file, ngi_op_t op, ngi_type_t type, void * out_buff, size_t off, size_t size){
|
||||||
|
fseek(file, off, SEEK_SET);
|
||||||
|
|
||||||
|
assert(type == NGI_TYPE_FLOAT);
|
||||||
|
assert(op == NGI_OP_MAX);
|
||||||
|
|
||||||
|
float * data = malloc(size);
|
||||||
|
fread(data, size, 1, file);
|
||||||
|
|
||||||
|
float mx = -INFINITY;
|
||||||
|
for(size_t i=0; i < size/sizeof(float); i++){
|
||||||
|
mx = data[i] > mx ? data[i]: mx;
|
||||||
|
}
|
||||||
|
*(float*) out_buff = mx;
|
||||||
|
|
||||||
|
free(data);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
int server_reduce_any_func(FILE * file, void (*func)(int8_t * data, void * out_buff, size_t off, size_t size), void * out_buff, size_t max_out_data_size, size_t off, size_t size){
|
int server_reduce_any_func(FILE * file, void (*func)(int8_t * data, void * out_buff, size_t off, size_t size), void * out_buff, size_t max_out_data_size, size_t off, size_t size){
|
||||||
fseek(file, off, SEEK_SET);
|
fseek(file, off, SEEK_SET);
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,12 @@
|
||||||
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
//#include "ngi_log.h"
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Define the size in B of file extent exchanged between server and clint
|
||||||
|
*/
|
||||||
|
#define NGI_EXT_SZ 1048576
|
||||||
|
|
||||||
typedef enum NGI_OP{
|
typedef enum NGI_OP{
|
||||||
NGI_OP_MAX
|
NGI_OP_MAX
|
||||||
|
@ -12,6 +18,12 @@ typedef enum NGI_TYPE{
|
||||||
NGI_TYPE_FLOAT
|
NGI_TYPE_FLOAT
|
||||||
} ngi_type_t;
|
} ngi_type_t;
|
||||||
|
|
||||||
|
/*
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
int ngi_server_side_reduce(FILE * , ngi_op_t , ngi_type_t , void * , size_t , size_t );
|
||||||
|
int ngi_client_side_reduce(FILE * , ngi_op_t , ngi_type_t , void * , size_t , size_t );
|
||||||
|
int ngi_reduce_stub(FILE * , ngi_op_t , ngi_type_t , void * , size_t , size_t );
|
||||||
/*
|
/*
|
||||||
* This function invoces a well known reduction operation on data of a well known type
|
* This function invoces a well known reduction operation on data of a well known type
|
||||||
* Return 0 upon success, any other code, if e.g., short read
|
* Return 0 upon success, any other code, if e.g., short read
|
||||||
|
|
|
@ -0,0 +1,42 @@
|
||||||
|
/******************************************************************************
|
||||||
|
*
|
||||||
|
* NGI log utilities
|
||||||
|
*
|
||||||
|
******************************************************************************/
|
||||||
|
|
||||||
|
#ifndef COMMON_NGI_LOG_H
|
||||||
|
#define COMMON_NGI_LOG_H
|
||||||
|
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
|
||||||
|
enum ngi_log_level
|
||||||
|
{
|
||||||
|
NGI_LOG_DBG,
|
||||||
|
NGI_LOG_WARN,
|
||||||
|
NGI_LOG_ERROR,
|
||||||
|
NGI_LOG_MAX
|
||||||
|
};
|
||||||
|
enum ngi_log_level NGI_Log_Level = NGI_LOG_DBG;
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
#define ngi_log(lvl, fmt, ...) \
|
||||||
|
do { if (lvl >= NGI_Log_Level) fprintf(stderr, "%s:%d:%s(): %s" fmt,\
|
||||||
|
__func__, __VA_ARGS__); } while (0)
|
||||||
|
*/
|
||||||
|
#define ngi_log(lvl, fmt, ...) \
|
||||||
|
do { if (lvl >= NGI_Log_Level ) fprintf(stderr, "== ngi_log == %s: " fmt, __func__, ## __VA_ARGS__);}\
|
||||||
|
while (0)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#define ngi_dbg(msg, ...) ngi_log(NGI_LOG_DBG, msg, ##__VA_ARGS__)
|
||||||
|
#define ngi_warn(msg, ...) ngi_log(NGI_LOG_WARN, msg, ##__VA_ARGS__)
|
||||||
|
#define ngi_error(msg, ...) ngi_log(NGI_LOG_ERROR, msg, ##__VA_ARGS__)
|
||||||
|
|
||||||
|
/* getter and setters */
|
||||||
|
#define ngi_log_level_set(ngi_log_level) NGI_Log_Level = ngi_log_level;
|
||||||
|
#define ngi_log_level_get(ngi_log_level) ngi_log_level;
|
||||||
|
|
||||||
|
#endif
|
|
@ -0,0 +1,43 @@
|
||||||
|
|
||||||
|
ROOT=../../server-side-compute
|
||||||
|
|
||||||
|
CC=gcc
|
||||||
|
CFLAGS=-Wall -Wextra -g3
|
||||||
|
INCLUDES=-I$(ROOT)/emulation/
|
||||||
|
RM = rm -rf
|
||||||
|
SRC=reduce-traditionnal.c data_gen.c
|
||||||
|
BIN=reduce-traditional data-gen reduce-ngi
|
||||||
|
OBJ=libngi.so libreduce-ngi-userlib.so
|
||||||
|
|
||||||
|
all: clean build test
|
||||||
|
|
||||||
|
build: $(BIN) $(OBJ)
|
||||||
|
|
||||||
|
clean:
|
||||||
|
$(RM) $(BIN) $(OBJ)
|
||||||
|
|
||||||
|
reduce-traditional: reduce-traditional.c
|
||||||
|
$(CC) $(CFLAGS) $< -o $@ -g3
|
||||||
|
|
||||||
|
data-gen: data-gen.c
|
||||||
|
$(CC) $(CFLAGS) $< -o $@
|
||||||
|
|
||||||
|
# Build NGI Emulation lib
|
||||||
|
libngi.so: $(ROOT)/emulation/ngi.c
|
||||||
|
@echo 'Building ngi shared library: $@'
|
||||||
|
$(CC) $(CLFAGS) $(INCLUDES) $< -shared -fpic -o $@
|
||||||
|
|
||||||
|
libreduce-ngi-userlib.so: reduce-ngi-userlib.c
|
||||||
|
@echo 'Building ngi user shared library: $@'
|
||||||
|
gcc $(CFLAGS) -Wall -Wextra $< -shared -fpic -o $@
|
||||||
|
|
||||||
|
reduce-ngi: reduce-ngi.c libngi.so
|
||||||
|
$(CC) $(CFLAGS) $< -o $@ $(INCLUDES) -L. -l ngi -Wl,-rpath=$(PWD) -ldl
|
||||||
|
|
||||||
|
test:
|
||||||
|
@echo "Data generation"
|
||||||
|
@./data-gen
|
||||||
|
@echo "Executing traditional"
|
||||||
|
@./reduce-traditional
|
||||||
|
@echo "Executing NGI"
|
||||||
|
@./reduce-ngi
|
|
@ -5,17 +5,28 @@
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <float.h>
|
#include <float.h>
|
||||||
|
#include <time.h>
|
||||||
|
#include <errno.h>
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Generate test data
|
* Generate test data
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
// generate float number between 0 and 99 with at most 2 decimal
|
||||||
|
float gen_float(void){
|
||||||
|
u_int32_t my_int = rand() % 10000;
|
||||||
|
float my_float = (float) my_int / 100;
|
||||||
|
return my_float;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// generate some data such that we can reduce it later
|
// generate some data such that we can reduce it later
|
||||||
void prepare_data(int size){
|
void prepare_data(int size){
|
||||||
float * data = malloc(size * sizeof(float));
|
float * data = malloc(size * sizeof(float));
|
||||||
for(int i=0; i < size; i++){
|
for(int i=0; i < size; i++){
|
||||||
data[i] = (float) i;
|
data[i] = (float) gen_float();
|
||||||
}
|
}
|
||||||
|
|
||||||
FILE * file = fopen("data.bin", "wb");
|
FILE * file = fopen("data.bin", "wb");
|
||||||
|
@ -25,8 +36,28 @@ void prepare_data(int size){
|
||||||
free(data);
|
free(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parse generated data
|
||||||
|
void parse_data(int size){
|
||||||
|
float data ;
|
||||||
|
|
||||||
|
FILE * file = fopen("data.bin", "r");
|
||||||
|
|
||||||
|
for(int i=0; i < size; i++){
|
||||||
|
int res = fread(&data, sizeof(float), 1, file);
|
||||||
|
if (res != 1){
|
||||||
|
res = errno;
|
||||||
|
fprintf(stderr, "fread failed to read the right amount of float: %s\n",strerror(res));
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
fprintf(stdout, "%.2f ", data);
|
||||||
|
}
|
||||||
|
|
||||||
|
fclose(file);
|
||||||
|
}
|
||||||
|
|
||||||
int main(){
|
int main(){
|
||||||
int size = 100;
|
int size = 40;
|
||||||
|
srand(time(NULL));
|
||||||
prepare_data(size);
|
prepare_data(size);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,6 +21,14 @@ void reduce_data(int size){
|
||||||
printf("Maximum: %f\n", mx);
|
printf("Maximum: %f\n", mx);
|
||||||
fclose(file);
|
fclose(file);
|
||||||
}
|
}
|
||||||
|
// read the data and reduce it using the NGI emulation lib
|
||||||
|
void reduce_data_native(int size){
|
||||||
|
FILE * file = fopen("data.bin", "rb");
|
||||||
|
float mx;
|
||||||
|
ngi_reduce_stub(file, NGI_OP_MAX, NGI_TYPE_FLOAT, & mx, 0, size*sizeof(float));
|
||||||
|
printf("Maximum (Ngi Native): %f\n", mx);
|
||||||
|
fclose(file);
|
||||||
|
}
|
||||||
|
|
||||||
void reduce_min_func(int8_t * data_i, void * out_buff, size_t off, size_t size){
|
void reduce_min_func(int8_t * data_i, void * out_buff, size_t off, size_t size){
|
||||||
float * outf = (float*) out_buff;
|
float * outf = (float*) out_buff;
|
||||||
|
@ -70,8 +78,9 @@ void reduce_data_func_lib(int size){
|
||||||
|
|
||||||
|
|
||||||
int main(){
|
int main(){
|
||||||
int size = 100;
|
int size = 40;
|
||||||
reduce_data(size);
|
reduce_data(size);
|
||||||
|
reduce_data_native(size);
|
||||||
reduce_data_func(size);
|
reduce_data_func(size);
|
||||||
reduce_data_func_lib(size);
|
reduce_data_func_lib(size);
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
@ -31,7 +31,7 @@ void reduce_data(int size){
|
||||||
|
|
||||||
|
|
||||||
int main(){
|
int main(){
|
||||||
int size = 100;
|
int size = 40;
|
||||||
reduce_data(size);
|
reduce_data(size);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue