cs
/
icp
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Uses docker to run.

This commit is contained in:
Julian M. Kunkel 2018-05-06 13:11:47 +01:00
parent 1c546f8d05
commit 7a0f783aa3
10 changed files with 31 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dev/Dockerfile
View File

@ -16,4 +16,9 @@ RUN sed -i "s#data_directory = '.*'#data_directory = '/data/run/postgres'#" /etc
RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/10/main/pg_hba.conf
RUN echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io libmpich-dev nano
RUN echo 'DOCKER_OPTS="-g /data/run/docker"' >> /etc/default/docker
RUN adduser --system --no-create-home --home /data --uid 1001 unpriv
RUN usermod -a -G docker www-data
CMD /data/dev/run-internal.sh

4
dev/run-container.sh
View File

@ -1,2 +1,4 @@
#!/bin/bash
docker run -p 127.0.0.1:8888:80 -h hps -it --rm -v $PWD/../:/data/ kunkel/oer
# Priviledged mode for running docker inside docker.
docker run -p 127.0.0.1:8888:80 -h hps -it --rm --privileged -v $PWD/../:/data/ kunkel/oer # -d

8
dev/run-internal.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash -e
adduser --system --no-create-home --home /data --uid 1000 www-user
sed -i "s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=www-user/" /etc/apache2/envvars
#adduser --system --no-create-home --home /data --uid 1000 www-user
#sed -i "s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=www-user/" /etc/apache2/envvars
sed -i "s#data_directory = '.*'#data_directory = '/data/run/postgres'#" /etc/postgresql/10/main/postgresql.conf
/etc/init.d/postgresql restart
@ -26,8 +26,12 @@ if [[ ! -e $V ]] ; then
pip3 install -U -r /data/dev/requirements.txt
pip3 install --ignore-installed python-social-auth
python3 ./manage.py migrate
# prepare docker image for worker
docker build -t kunkel/oer-worker /data/dev/worker
fi
source $V/bin/activate
/etc/init.d/docker start
/bin/bash

2
dev/worker/Dockerfile
View File

@ -2,3 +2,5 @@ FROM ubuntu:18.04
WORKDIR /data
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y mpich
RUN adduser --system --no-create-home --home /data --uid 1001 unpriv

3
dev/worker/run-container.sh
View File

@ -1,2 +1,3 @@
#!/bin/bash
docker run -h oer-worker -it --rm -v $PWD/../:/data/ kunkel/oer-worker /bin/bash
# EXAMPLE:
docker run -h oer-worker --user 1001:65534 -it --rm -v /data/run/jobs/10/:/data/ kunkel/oer-worker /data/program

13
oer/exercise-formats/program/match-regex-parallel/controller.py
View File

@ -16,7 +16,6 @@ import time
def build(ctx):
# Add filenames
allArgs = ['/usr/bin/make']
#, self.sourcefile, "-o", "%s.exe" % self.sourcefile]
# start subprocess in with work_path as cwd
p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path)
@ -45,14 +44,8 @@ def execute(ctx):
errs = errs.decode("utf-8")
msgs = msgs.decode("utf-8")
ctx.execute_result = (retval, errs, msgs)
#ctx.celery_task.update_state(state='SUBMITTED TO SLURM', meta=response(ctx))
#print("parallel execute(), before sleep", file=sys.stderr)
#time.sleep(20)
#print("parallel execute(), after sleep", file=sys.stderr)
retval = ctx.get_file_content('job.exit')
errs = ctx.get_file_content('job.err')
@ -85,12 +78,10 @@ def grade(ctx):
def response(ctx):
output = ""
if ctx.build_result[0] != 0:
output = ctx.build_result[1]
else:
output = ctx.execute_result[2]
output = ctx.execute_result[2] + "\n" + ctx.execute_result[1]
grade = ctx.grade_result
msg = ""

12
oer/exercise-formats/program/match-regex/controller.py
View File

@ -16,7 +16,6 @@ import time
def build(ctx):
# Add filenames
allArgs = ['/usr/bin/make']
#, self.sourcefile, "-o", "%s.exe" % self.sourcefile]
# start subprocess in with work_path as cwd
p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path)
@ -34,7 +33,8 @@ def execute(ctx):
if ctx.build_result[0] != 0:
return
allArgs = [ctx.programfile]
allArgs = ("/usr/bin/docker run -h oer-worker --user 1001:65534 --rm -v /data/run/jobs/%s/:/data/ kunkel/oer-worker /data/program" % ctx.id).split(" ")
#print(" ".join(allArgs), file=sys.stderr)
p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path)
timeout = 1
@ -53,6 +53,7 @@ def execute(ctx):
errs = errs.decode("utf-8")
msgs = msgs.decode("utf-8")
#print(errs, file=sys.stderr)
return retval, errs, msgs
@ -66,8 +67,8 @@ def grade(ctx):
ctx.execute_result = execute(ctx)
print(ctx.data['data']['regex'], file=sys.stderr)
print(ctx.execute_result, file=sys.stderr)
#print(ctx.data['data']['regex'], file=sys.stderr)
#print(ctx.execute_result, file=sys.stderr)
if re.match(ctx.data['data']['regex'], ctx.execute_result[2]) != None:
return 'PASS'
else:
@ -75,11 +76,10 @@ def grade(ctx):
def response(ctx):
output = ""
if ctx.build_result[0] != 0:
output = ctx.build_result[1]
else:
output = ctx.execute_result[2]
output = ctx.execute_result[2] + "\n" + ctx.execute_result[1]
grade = ctx.grade_result
msg = ""

12
oer/exercise-formats/program/scriptgrade/controller.py
View File

@ -14,7 +14,6 @@ import time
def build(ctx):
# Add filenames
allArgs = ['/usr/bin/make']
#, self.sourcefile, "-o", "%s.exe" % self.sourcefile]
# start subprocess in with work_path as cwd
p = subprocess.Popen(allArgs, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ctx.work_path)
@ -61,7 +60,6 @@ def grade(ctx):
print(ctx.execute_result, file=sys.stderr)
allArgs = ['/usr/bin/make', 'execute']
allArgs = ['./test.sh']
# start subprocess in with work_path as cwd
@ -94,18 +92,10 @@ def grade(ctx):
def response(ctx):
output = ""
print(ctx.build_result, file=sys.stderr)
print(ctx.execute_result, file=sys.stderr)
print(ctx.grade_result, file=sys.stderr)
if ctx.build_result[0] != 0:
output = ctx.build_result[1]
else:
output = ctx.execute_result[2]
output = ctx.execute_result[2] + "\n" + ctx.execute_result[1]
grade = ctx.grade_result
msg = ""

2
src/rest/views.py
View File

@ -63,7 +63,7 @@ def local_execute_result(id):
with open(dir + "job.json") as json_data:
d = json.load(json_data)
# cleanup of outdated directories
shutil.rmtree(dir)
#shutil.rmtree(dir)
return {"status" : 'SUCCESS', "result": d["response"]}
else:

7
src/runner/tasks.py
View File

@ -14,7 +14,6 @@ WORK_DIR = "/data/run/jobs"
OER_FORMATS_PATH = "/data/oer/exercise-formats/"
def load_handler(path):
handler = {'name': None, 'loader': None, 'mod': None}
#print(path, file=sys.stderr)
for root, dirs, files in os.walk(path):
@ -23,13 +22,13 @@ def load_handler(path):
if 'controller.py' in files:
print("INFO: controller.py present for '%s'!" % path, file=sys.stderr)
try:
#name = root.replace(path, '')
name = path
loader = importlib.machinery.SourceFileLoader(name, root + '/controller.py')
mod = loader.load_module()
handler = {'name': name, 'loader': loader, 'mod': mod}
except:
except Exception as e:
print("WARNING: Loading controller failed for '%s'!" % path, file=sys.stderr)
print("Error %s" % s, file=sys.stderr)
else:
print("INFO: No controller found for '%s'!" % path, file=sys.stderr)
@ -58,6 +57,7 @@ class Job(object):
def __init__(self, job):
#print("Job.__init__()", file=sys.stderr)
self.data = job
self.id = job['_id']
self.work_path = WORK_DIR + '/' + str(job['_id']) + '/'
self.handler_format = {'name': None, 'loader': None, 'mod': None}
self.handler_exercise = {'name': None, 'loader': None, 'mod': None}
@ -93,7 +93,6 @@ class Job(object):
shutil.copytree(src, dst)
self.sourcefile = self.work_path + 'program.c'
self.programfile = self.work_path + 'program'
# overwrite default with user submission
source = open(self.sourcefile, 'w')