cs
/
icp
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Uses docker to run.

This commit is contained in:
Julian M. Kunkel 2018-05-06 13:11:47 +01:00
parent 1c546f8d05
commit 7a0f783aa3
10 changed files with 31 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dev/Dockerfile
View File

@ -16,4 +16,9 @@ RUN sed -i "s#data_directory = '.*'#data_directory = '/data/run/postgres'#" /etc
RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/10/main/pg_hba.conf RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/10/main/pg_hba.conf
RUN echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf RUN echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io libmpich-dev nano
RUN echo 'DOCKER_OPTS="-g /data/run/docker"' >> /etc/default/docker
RUN adduser --system --no-create-home --home /data --uid 1001 unpriv
RUN usermod -a -G docker www-data
CMD /data/dev/run-internal.sh CMD /data/dev/run-internal.sh

4
dev/run-container.sh
View File

@ -1,2 +1,4 @@
#!/bin/bash #!/bin/bash
docker run -p 127.0.0.1:8888:80 -h hps -it --rm -v $PWD/../:/data/ kunkel/oer
# Priviledged mode for running docker inside docker.
docker run -p 127.0.0.1:8888:80 -h hps -it --rm --privileged -v $PWD/../:/data/ kunkel/oer # -d

8
dev/run-internal.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash -e #!/bin/bash -e
adduser --system --no-create-home --home /data --uid 1000 www-user #adduser --system --no-create-home --home /data --uid 1000 www-user
sed -i "s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=www-user/" /etc/apache2/envvars #sed -i "s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=www-user/" /etc/apache2/envvars
sed -i "s#data_directory = '.*'#data_directory = '/data/run/postgres'#" /etc/postgresql/10/main/postgresql.conf sed -i "s#data_directory = '.*'#data_directory = '/data/run/postgres'#" /etc/postgresql/10/main/postgresql.conf
/etc/init.d/postgresql restart /etc/init.d/postgresql restart
@ -26,8 +26,12 @@ if [[ ! -e $V ]] ; then
pip3 install -U -r /data/dev/requirements.txt pip3 install -U -r /data/dev/requirements.txt
pip3 install --ignore-installed python-social-auth pip3 install --ignore-installed python-social-auth
python3 ./manage.py migrate python3 ./manage.py migrate
# prepare docker image for worker
docker build -t kunkel/oer-worker /data/dev/worker
fi fi
source $V/bin/activate source $V/bin/activate
/etc/init.d/docker start
/bin/bash /bin/bash

2
dev/worker/Dockerfile
View File

@ -2,3 +2,5 @@ FROM ubuntu:18.04
WORKDIR /data WORKDIR /data
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y mpich
RUN adduser --system --no-create-home --home /data --uid 1001 unpriv

3
dev/worker/run-container.sh
View File

@ -1,2 +1,3 @@
#!/bin/bash #!/bin/bash
docker run -h oer-worker -it --rm -v $PWD/../:/data/ kunkel/oer-worker /bin/bash # EXAMPLE:
docker run -h oer-worker --user 1001:65534 -it --rm -v /data/run/jobs/10/:/data/ kunkel/oer-worker /data/program

11
oer/exercise-formats/program/match-regex-parallel/controller.py
View File

@ -16,7 +16,6 @@ import time
def build(ctx): def build(ctx):
# Add filenames # Add filenames
allArgs = ['/usr/bin/make'] allArgs = ['/usr/bin/make']
#, self.sourcefile, "-o", "%s.exe" % self.sourcefile]
# start subprocess in with work_path as cwd # start subprocess in with work_path as cwd
p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path) p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path)
@ -47,12 +46,6 @@ def execute(ctx):
msgs = msgs.decode("utf-8") msgs = msgs.decode("utf-8")
ctx.execute_result = (retval, errs, msgs) ctx.execute_result = (retval, errs, msgs)
#ctx.celery_task.update_state(state='SUBMITTED TO SLURM', meta=response(ctx))
#print("parallel execute(), before sleep", file=sys.stderr)
#time.sleep(20)
#print("parallel execute(), after sleep", file=sys.stderr)
retval = ctx.get_file_content('job.exit') retval = ctx.get_file_content('job.exit')
errs = ctx.get_file_content('job.err') errs = ctx.get_file_content('job.err')
@ -85,12 +78,10 @@ def grade(ctx):
def response(ctx): def response(ctx):
output = ""
if ctx.build_result[0] != 0: if ctx.build_result[0] != 0:
output = ctx.build_result[1] output = ctx.build_result[1]
else: else:
output = ctx.execute_result[2] output = ctx.execute_result[2] + "\n" + ctx.execute_result[1]
grade = ctx.grade_result grade = ctx.grade_result
msg = "" msg = ""

12
oer/exercise-formats/program/match-regex/controller.py
View File

@ -16,7 +16,6 @@ import time
def build(ctx): def build(ctx):
# Add filenames # Add filenames
allArgs = ['/usr/bin/make'] allArgs = ['/usr/bin/make']
#, self.sourcefile, "-o", "%s.exe" % self.sourcefile]
# start subprocess in with work_path as cwd # start subprocess in with work_path as cwd
p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path) p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path)
@ -34,7 +33,8 @@ def execute(ctx):
if ctx.build_result[0] != 0: if ctx.build_result[0] != 0:
return return
allArgs = [ctx.programfile] allArgs = ("/usr/bin/docker run -h oer-worker --user 1001:65534 --rm -v /data/run/jobs/%s/:/data/ kunkel/oer-worker /data/program" % ctx.id).split(" ")
#print(" ".join(allArgs), file=sys.stderr)
p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path) p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path)
timeout = 1 timeout = 1
@ -53,6 +53,7 @@ def execute(ctx):
errs = errs.decode("utf-8") errs = errs.decode("utf-8")
msgs = msgs.decode("utf-8") msgs = msgs.decode("utf-8")
#print(errs, file=sys.stderr)
return retval, errs, msgs return retval, errs, msgs
@ -66,8 +67,8 @@ def grade(ctx):
ctx.execute_result = execute(ctx) ctx.execute_result = execute(ctx)
print(ctx.data['data']['regex'], file=sys.stderr) #print(ctx.data['data']['regex'], file=sys.stderr)
print(ctx.execute_result, file=sys.stderr) #print(ctx.execute_result, file=sys.stderr)
if re.match(ctx.data['data']['regex'], ctx.execute_result[2]) != None: if re.match(ctx.data['data']['regex'], ctx.execute_result[2]) != None:
return 'PASS' return 'PASS'
else: else:
@ -75,11 +76,10 @@ def grade(ctx):
def response(ctx): def response(ctx):
output = ""
if ctx.build_result[0] != 0: if ctx.build_result[0] != 0:
output = ctx.build_result[1] output = ctx.build_result[1]
else: else:
output = ctx.execute_result[2] output = ctx.execute_result[2] + "\n" + ctx.execute_result[1]
grade = ctx.grade_result grade = ctx.grade_result
msg = "" msg = ""

12
oer/exercise-formats/program/scriptgrade/controller.py
View File

@ -14,7 +14,6 @@ import time
def build(ctx): def build(ctx):
# Add filenames # Add filenames
allArgs = ['/usr/bin/make'] allArgs = ['/usr/bin/make']
#, self.sourcefile, "-o", "%s.exe" % self.sourcefile]
# start subprocess in with work_path as cwd # start subprocess in with work_path as cwd
p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path) p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path)
@ -61,7 +60,6 @@ def grade(ctx):
print(ctx.execute_result, file=sys.stderr) print(ctx.execute_result, file=sys.stderr)
allArgs = ['/usr/bin/make', 'execute']
allArgs = ['./test.sh'] allArgs = ['./test.sh']
# start subprocess in with work_path as cwd # start subprocess in with work_path as cwd
@ -94,18 +92,10 @@ def grade(ctx):
def response(ctx): def response(ctx):
output = ""
print(ctx.build_result, file=sys.stderr)
print(ctx.execute_result, file=sys.stderr)
print(ctx.grade_result, file=sys.stderr)
if ctx.build_result[0] != 0: if ctx.build_result[0] != 0:
output = ctx.build_result[1] output = ctx.build_result[1]
else: else:
output = ctx.execute_result[2] output = ctx.execute_result[2] + "\n" + ctx.execute_result[1]
grade = ctx.grade_result grade = ctx.grade_result
msg = "" msg = ""

2
src/rest/views.py
View File

@ -63,7 +63,7 @@ def local_execute_result(id):
with open(dir + "job.json") as json_data: with open(dir + "job.json") as json_data:
d = json.load(json_data) d = json.load(json_data)
# cleanup of outdated directories # cleanup of outdated directories
shutil.rmtree(dir) #shutil.rmtree(dir)
return {"status" : 'SUCCESS', "result": d["response"]} return {"status" : 'SUCCESS', "result": d["response"]}
else: else:

7
src/runner/tasks.py
View File

@ -14,7 +14,6 @@ WORK_DIR = "/data/run/jobs"
OER_FORMATS_PATH = "/data/oer/exercise-formats/" OER_FORMATS_PATH = "/data/oer/exercise-formats/"
def load_handler(path): def load_handler(path):
handler = {'name': None, 'loader': None, 'mod': None} handler = {'name': None, 'loader': None, 'mod': None}
#print(path, file=sys.stderr) #print(path, file=sys.stderr)
for root, dirs, files in os.walk(path): for root, dirs, files in os.walk(path):
@ -23,13 +22,13 @@ def load_handler(path):
if 'controller.py' in files: if 'controller.py' in files:
print("INFO: controller.py present for '%s'!" % path, file=sys.stderr) print("INFO: controller.py present for '%s'!" % path, file=sys.stderr)
try: try:
#name = root.replace(path, '')
name = path name = path
loader = importlib.machinery.SourceFileLoader(name, root + '/controller.py') loader = importlib.machinery.SourceFileLoader(name, root + '/controller.py')
mod = loader.load_module() mod = loader.load_module()
handler = {'name': name, 'loader': loader, 'mod': mod} handler = {'name': name, 'loader': loader, 'mod': mod}
except: except Exception as e:
print("WARNING: Loading controller failed for '%s'!" % path, file=sys.stderr) print("WARNING: Loading controller failed for '%s'!" % path, file=sys.stderr)
print("Error %s" % s, file=sys.stderr)
else: else:
print("INFO: No controller found for '%s'!" % path, file=sys.stderr) print("INFO: No controller found for '%s'!" % path, file=sys.stderr)
@ -58,6 +57,7 @@ class Job(object):
def __init__(self, job): def __init__(self, job):
#print("Job.__init__()", file=sys.stderr) #print("Job.__init__()", file=sys.stderr)
self.data = job self.data = job
self.id = job['_id']
self.work_path = WORK_DIR + '/' + str(job['_id']) + '/' self.work_path = WORK_DIR + '/' + str(job['_id']) + '/'
self.handler_format = {'name': None, 'loader': None, 'mod': None} self.handler_format = {'name': None, 'loader': None, 'mod': None}
self.handler_exercise = {'name': None, 'loader': None, 'mod': None} self.handler_exercise = {'name': None, 'loader': None, 'mod': None}
@ -93,7 +93,6 @@ class Job(object):
shutil.copytree(src, dst) shutil.copytree(src, dst)
self.sourcefile = self.work_path + 'program.c' self.sourcefile = self.work_path + 'program.c'
self.programfile = self.work_path + 'program'
# overwrite default with user submission # overwrite default with user submission
source = open(self.sourcefile, 'w') source = open(self.sourcefile, 'w')