diff --git a/Dockerfile b/Dockerfile index c4a2e62..d9faccf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ ADD ./lib/* ./lib/ ADD ./macros/* /simobot/macros/ ADD ./resources/* /simobot/resources/ ADD ./features/* /simobot/features/ -ADD addHost.sh backupSimo.sh dumpExpl.sh .gitignore joindb.js LightweightApi.py main.js marker migrate-logs.js package.json README.md redis.conf repeatSimo settings.json.example settings_pythonsimo.cfg settings_pythonsimo.cfg.example simojs.sqlite telegraf.conf ./ +ADD .gitignore joindb.js LightweightApi.py main.js migrate-logs.js package.json README.md redis.conf settings.json.example settings_pythonsimo.cfg settings_pythonsimo.cfg.example simojs.sqlite telegraf.conf ./ ADD ./templates/* /simobot/templates/ RUN chown -R simobot:simobot /simobot diff --git a/addHost.sh b/addHost.sh deleted file mode 100755 index bd56d23..0000000 --- a/addHost.sh +++ /dev/null @@ -1,6 +0,0 @@ -HOST_DOMAIN="host.docker.internal" -ping -q -c1 $HOST_DOMAIN > /dev/null 2>&1 -if [ $? -ne 0 ]; then - HOST_IP=$(ip route | awk 'NR==1 {print $3}') - echo -e "$HOST_IP\t$HOST_DOMAIN" >> /etc/hosts -fi diff --git a/features/ml-simo.js b/features/ml-simo.js deleted file mode 100644 index 9a08106..0000000 --- a/features/ml-simo.js +++ /dev/null @@ -1,89 +0,0 @@ -const axios = require('axios').default; -const { v4: uuidv4 } = require('uuid'); - - -function fixResult(r) { - let result = Object.assign({}, r.data) - result.output = result.output.replace('\n', '|').substr(0, 300) - result.filename = 'http://gpt.prototyping.xyz/' + result.filename - return result -} - -var gptfi = function(client, channel, from, line) { - var url = "http://ml-simo:8765/gpt"; - - const msg = line.split(' ').slice(1).join(' ') - - const payload = { - prompt: msg, - translate: true - } - - axios.post(url, payload).then(r => { // should convert to json automatically - console.log('recv gpt') - let output = fixResult(r) - //client.say(channel, JSON.stringify(output)); - client.say(channel, output.filename + " " + output.output); - }) -} - -var gpt = function(client, channel, from, line) { - var url = "http://ml-simo:8765/gpt"; - - const msg = line.split(' ').slice(1).join(' ') - - const payload = { - prompt: msg - } - - axios.post(url, payload).then(r => { // should convert to json automatically - console.log('recv gpt') - let output = fixResult(r) - //client.say(channel, JSON.stringify(output)); - client.say(channel, output.filename + " " + output.output); - }) -} - -var gptNoPrompt = function(client, channel, from, line) { - var url = "http://ml-simo:8765/gpt"; - - const msg = line.split(' ').slice(1).join(' ') - - const payload = { - prompt: msg - } - - axios.post(url, payload).then(r => { // should convert to json automatically - console.log('recv gpt') - let output = fixResult(r) - client.say(channel, output.output.replace(msg, "")); - }) -} - -var gptJson = function(client, channel, from, line) { - var url = "http://ml-simo:8765/gpt"; - const msg = line.split(' ').slice(1).join(' ') - let payload = {} - try { - payload = JSON.parse(msg) // should convert automatically - } - catch(e) { - client.say(channel, "JSON parse crashed") - } - axios.post(url, payload).then(r => { - console.log('recv gpt') - let output = fixResult(r) - client.say(channel, JSON.stringify(output)); - }) -} - -module.exports = { - name: "gpt", //not required atm iirc - commands: {} - //commands: { - // "!gpt": gpt, - // "!gptfi": gptfi, - // "!gptnoprompt": gptNoPrompt, - // "!gptjson": gptJson - //} -} diff --git a/marker b/marker deleted file mode 100644 index 23ad314..0000000 --- a/marker +++ /dev/null @@ -1 +0,0 @@ -Sun Sep 22 18:34:34 CEST 2019 diff --git a/ml-simo/.gitignore b/ml-simo/.gitignore deleted file mode 100644 index 2048fd9..0000000 --- a/ml-simo/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -mlsimo-data -__pycache__ -mlsimo-data diff --git a/ml-simo/Dockerfile_mlsimo b/ml-simo/Dockerfile_mlsimo deleted file mode 100644 index c755557..0000000 --- a/ml-simo/Dockerfile_mlsimo +++ /dev/null @@ -1,17 +0,0 @@ -FROM huggingface/transformers-pytorch-gpu - -WORKDIR /mlsimo - -COPY requirements.txt /mlsimo/ -RUN pip install -r requirements.txt -COPY server.py /mlsimo/ - -EXPOSE 8765 -#CMD python3 server.py -CMD gunicorn -w 1 --timeout 3000 server:app - -#RUN apk --no-cache --update add git build-base -#RUN git clone https://github.com/code-n-beer/SimoBotPython /pythonsimo -#COPY pythonsimo-data/* /pythonsimo/Resources/ -#EXPOSE 8888 -#CMD python main.py diff --git a/ml-simo/docker-compose.yml b/ml-simo/docker-compose.yml deleted file mode 100644 index a49862f..0000000 --- a/ml-simo/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '2' -services: - mlsimo: - build: - context: . - dockerfile: Dockerfile_mlsimo - expose: - - "8000" - ports: - - "8765:8000" - restart: always - volumes: - - "./mlsimo-data/:/mlsimo-data" - environment: - - NVIDIA_VISIBLE_DEVICES=all - - HF_HOME=/mlsimo-data/ - runtime: nvidia \ No newline at end of file diff --git a/ml-simo/requirements.txt b/ml-simo/requirements.txt deleted file mode 100644 index 9f81229..0000000 --- a/ml-simo/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -shortuuid -flask -transformers -gunicorn \ No newline at end of file diff --git a/ml-simo/server.py b/ml-simo/server.py deleted file mode 100644 index a22b0cf..0000000 --- a/ml-simo/server.py +++ /dev/null @@ -1,109 +0,0 @@ -from flask import Flask, request, jsonify -app = Flask(__name__) -from transformers import pipeline, PreTrainedModel, GPTNeoForCausalLM -from pprint import pprint -import torch -import shortuuid - -torch.zeros(10, 10).to('cuda:0') # early check that GPU works - -default_settings = { - "do_sample":True, - "min_length":40, - "max_length":500, - "length_penalty":1.0, -} - -#print('starting models') -#generator = pipeline('text-generation', model='EleutherAI/gpt-neo-1.3B', device=0) -#print('model loaded') - -if __name__ != '__main__': - print('not main, starting models') - generator = pipeline('text-generation', model='EleutherAI/gpt-neo-1.3B', device=0) - print('model loaded') -#else: -# print('main, skip models') -# generator = None - - -def translate_to_fi(text): - path = '/translate' - constructed_url = endpoint + path - params = { - 'api-version': '3.0', - 'from': 'en', - 'to': ['fi'] - } - constructed_url = endpoint + path - headers = { - 'Ocp-Apim-Subscription-Key': subscription_key, - 'Ocp-Apim-Subscription-Region': location, - 'Content-type': 'application/json', - 'X-ClientTraceId': str(uuid.uuid4()) - } - # You can pass more than one object in body. - body = [{ - 'text': text - }] - request = requests.post(constructed_url, params=params, headers=headers, json=body) - response = request.json() - return response[0]['translations'][0]['text'] - -def write_to_file(input_str): - uuid = shortuuid.uuid() - - if "" in input_str.lower(): - filename = "%s.html" % uuid - else: - filename = "%s.txt" % uuid - - with open('/mlsimo-data/output/' + filename, "w", encoding="utf8") as text_file: - text_file.write(input_str) - - return filename - -def gen(prompt, bad_words, settings, translate=False): - if settings is None: - settings = default_settings - if 'max_length' in settings and settings['max_length'] > 20000: - settings['max_length'] = 20000 - - if bad_words and len(bad_words) > 0: - ebun = generator(prompt, **settings, - bad_words_ids=generator.tokenizer(bad_words)['input_ids']) - else: - ebun = generator(prompt, **settings) - - output = ebun[0]['generated_text'] - if translate: - output = translate_to_fi(output) - filename = write_to_file(output) - - return filename, output - - -print('starting server') - -@app.route('/') -def index(): - return 'Server Works!' - -@app.route('/gpt', methods=['POST']) -def add_message(): - content = request.json - prompt = content['prompt'].replace("|","\n") - bad_words = content['bad_words'] if 'bad_words' in content else None - settings = content['settings'] if 'settings' in content else None - translate = True if 'translate' in content else False - - filename, output = gen(prompt, bad_words, settings, translate) - - return jsonify({'filename': filename, 'output': output}) - -#if __name__ == '__main__': -# app.run(host= '0.0.0.0', port=8000, debug=False, threaded=False, processes=0) -# print('server started, 0.0.0.0:8000') - - - diff --git a/pybase/Dockerfile_python_base b/pybase/Dockerfile_python_base deleted file mode 100644 index ffab49c..0000000 --- a/pybase/Dockerfile_python_base +++ /dev/null @@ -1,8 +0,0 @@ -FROM glavin001/alpine-python2-numpy-scipy - -RUN apk --no-cache --update add git build-base - -RUN git clone https://github.com/code-n-beer/SimoBotPython /pythonsimodeps -WORKDIR /pythonsimodeps - -RUN pip install -r requirements.txt diff --git a/repeatSimo b/repeatSimo deleted file mode 100755 index 220b2e2..0000000 --- a/repeatSimo +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -#authbind --deep mailin --webhook http://localhost/incoming_emails & -while [ 1 ]; do - echo "Kill dangling instances"; - if pgrep node; then - kill -9 $(pgrep node||); - fi - echo "Starting new instance of SimoJS"; - NODE_TLS_REJECT_UNAUTHORIZED=0 node main.js; -done diff --git a/backupSimo.sh b/scripts/backupSimo.sh similarity index 100% rename from backupSimo.sh rename to scripts/backupSimo.sh diff --git a/dumpExpl.sh b/scripts/dumpExpl.sh similarity index 100% rename from dumpExpl.sh rename to scripts/dumpExpl.sh diff --git a/rebuildSimo.sh b/scripts/rebuildSimo.sh similarity index 100% rename from rebuildSimo.sh rename to scripts/rebuildSimo.sh diff --git a/virtual_private_simo/.dockerignore b/virtual_private_simo/.dockerignore deleted file mode 100644 index a594077..0000000 --- a/virtual_private_simo/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -rebootServer.js diff --git a/virtual_private_simo/Dockerfile b/virtual_private_simo/Dockerfile deleted file mode 100644 index 192c869..0000000 --- a/virtual_private_simo/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -#FROM ubuntu:xenial -#FROM hypriot/rpi-node -FROM arm32v6/node:10.12-alpine - -#RUN apt-get update -#RUN apt-get install -y sudo curl wget telnet iputils-ping ghc build-essential python - -#RUN curl -sL https://deb.nodesource.com/setup_7.x | sudo -E bash - -#RUN sudo apt-get install -y nodejs -RUN echo http://dl-2.alpinelinux.org/alpine/edge/community/ >> /etc/apk/repositories - -RUN apk add --no-cache shadow sudo curl wget iputils - - -RUN mkdir -p /home/simobot - -ADD . /home/simobot - -RUN groupadd -r simobot && useradd -r -g simobot simobot - -RUN echo "simobot ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/default; - -RUN chown -R simobot:simobot /home/simobot - -USER simobot - -WORKDIR /home/simobot - -RUN npm install diff --git a/virtual_private_simo/index.js b/virtual_private_simo/index.js deleted file mode 100644 index 7a18592..0000000 --- a/virtual_private_simo/index.js +++ /dev/null @@ -1,27 +0,0 @@ -const serializeError = require('serialize-error'); -const express = require('express') -const app = express() -const bodyParser = require('body-parser') - -app.use(bodyParser.urlencoded({ extended: false })) -app.use(bodyParser.json()) - -app.post('/run', function (req, res) { - console.log('got a new command') - const exec = require('child_process').exec - const cmd = req.body.command - console.log(`got ${req.body.command}`) - - exec(cmd, function(error, stdout, stderr) { - let errormsg; - if(error) { - errormsg = serializeError(error) - } - console.log(errormsg, stdout, stderr) - res.send({error: errormsg, stdout, stderr}) - }) -}) - -app.listen(3500, function () { - console.log('Virtual simo started!') -}) diff --git a/virtual_private_simo/package.json b/virtual_private_simo/package.json deleted file mode 100644 index 5e8ea8b..0000000 --- a/virtual_private_simo/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "virtual_private_simo", - "version": "1.0.0", - "description": "virtual simo playground", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "MIT", - "dependencies": { - "body-parser": "^1.17.1", - "express": "^4.15.2", - "serialize-error": "^2.1.0" - } -} diff --git a/virtual_private_simo/rebootServer.js b/virtual_private_simo/rebootServer.js deleted file mode 100644 index c9de650..0000000 --- a/virtual_private_simo/rebootServer.js +++ /dev/null @@ -1,27 +0,0 @@ -const serializeError = require('serialize-error'); -const express = require('express') -const app = express() -const bodyParser = require('body-parser') - -app.use(bodyParser.urlencoded({ extended: false })) -app.use(bodyParser.json()) - -app.post('/reboot', function (req, res) { - console.log('got a new command') - const exec = require('child_process').exec - //const cmd = req.body.command - //console.log(`got ${req.body.command}`) - - exec('./reboot_virtualsimo', function(error, stdout, stderr) { - let errormsg; - if(error) { - errormsg = serializeError(error) - } - console.log(errormsg, stdout, stderr) - res.send({error: errormsg, stdout, stderr}) - }) -}) - -app.listen(3501, function () { - console.log('Virtual simo started!') -}) diff --git a/virtual_private_simo/reboot_virtualsimo b/virtual_private_simo/reboot_virtualsimo deleted file mode 100755 index aec6c7b..0000000 --- a/virtual_private_simo/reboot_virtualsimo +++ /dev/null @@ -1 +0,0 @@ -docker-compose build virtualsimo && docker-compose rm -f virtualsimo && docker-compose up virtualsimo