Skip to content

T
TA-Kel-16
Commit 5958c58f by Kyrie Cettyara
Options

initial commit

parent 52517dbd
Showing with 400 additions and 0 deletions
run/Dockerfile 0 → 100644
FROM ubuntu:14.04
MAINTAINER Vishal Lall "vishal.lall@caci.com"
LABEL Description="Install Moses SMT API" Version="1.2"
EXPOSE 5000
RUN apt-get update && apt-get install -y \
automake \
build-essential \
curl \
g++ \
git \
graphviz \
imagemagick \
libboost-all-dev \
libbz2-dev \
libgoogle-perftools-dev \
liblzma-dev \
libtool \
make \
python-dev \
python-pip \
python-yaml \
subversion \
unzip \
wget \
zlib1g-dev
RUN pip install \
flask \
flask-api
RUN mkdir -p /home/moses
WORKDIR /home/moses
RUN git clone https://github.com/moses-smt/mosesdecoder
RUN mkdir moses-models
# Giza
RUN wget -O giza-pp.zip "http://github.com/moses-smt/giza-pp/archive/master.zip"
RUN unzip giza-pp.zip
RUN rm giza-pp.zip
RUN mv giza-pp-master giza-pp
WORKDIR /home/moses/giza-pp
RUN make
WORKDIR /home/moses
RUN mkdir external-bin-dir
RUN cp giza-pp/GIZA++-v2/GIZA++ external-bin-dir
RUN cp giza-pp/GIZA++-v2/snt2cooc.out external-bin-dir
RUN cp giza-pp/mkcls-v2/mkcls external-bin-dir
# CMPH
RUN wget -O cmph-2.0.tar.gz "http://downloads.sourceforge.net/project/cmph/cmph/cmph-2.0.tar.gz?r=&ts=1426574097&use_mirror=cznic"
RUN tar zxvf cmph-2.0.tar.gz
WORKDIR /home/moses/cmph-2.0
RUN ./configure
RUN make
RUN make install
# IRSTLM
WORKDIR /home/moses
RUN wget -O irstlm-5.80.08.tgz "http://downloads.sourceforge.net/project/irstlm/irstlm/irstlm-5.80/irstlm-5.80.08.tgz?r=&ts=1342430877&use_mirror=kent"
RUN tar zxvf irstlm-5.80.08.tgz
WORKDIR /home/moses/irstlm-5.80.08/trunk
RUN /bin/bash -c "source regenerate-makefiles.sh"
RUN ./configure -prefix=/home/moses/irstlm
RUN make
RUN make install
WORKDIR /home/moses
ENV IRSTLM /home/moses/irstlm
# Get Newest Boost
RUN mkdir /home/moses/Downloads
WORKDIR /home/moses/Downloads
RUN wget https://sourceforge.net/projects/boost/files/boost/1.60.0/boost_1_60_0.tar.gz
RUN tar xf boost_1_60_0.tar.gz
RUN rm boost_1_60_0.tar.gz
WORKDIR boost_1_60_0/
RUN ./bootstrap.sh
RUN ./b2 -j4 --prefix=$PWD --libdir=$PWD/lib64 --layout=system link=static install || echo FAILURE
# Buckwalter for Arabic
WORKDIR /home/moses/Downloads
RUN wget http://search.cpan.org/CPAN/authors/id/G/GR/GRAFF/Encode-Buckwalter-1.1.tar.gz
RUN tar xf Encode-Buckwalter-1.1.tar.gz
RUN rm Encode-Buckwalter-1.1.tar.gz
WORKDIR /home/moses/Downloads/Encode-Buckwalter-1.1
RUN perl Makefile.PL
RUN make
RUN make test
RUN make install
# Get Flask API
WORKDIR /home/moses/Downloads
RUN git clone https://github.com/vlall/moses-api
# Download sample model
WORKDIR /home/moses/moses-models
RUN wget http://www.statmt.org/moses/download/sample-models.tgz
RUN tar xf sample-models.tgz
RUN rm sample-models.tgz
WORKDIR /home/moses/mosesdecoder
# COMPILE MOSES (Takes awhile...)
#RUN ./bjam --with-boost=/home/moses/Downloads/boost_1_60_0 --with-cmph=/home/moses/cmph-2.0 --with-irstlm=/home/moses/irstlm -j12
#WORKDIR /home/moses/
run/README.md 0 → 100644
[![Build Status](https://travis-ci.org/vlall/Moses-API.svg?branch=master)](https://travis-ci.org/vlall/Moses-API)
# Moses-API
Build a RESTful API server running your own machine translation model
### Instructions
- First pull the Docker image
```$ docker pull vlall/moses-api```
- Next, run the image and forward from 5000 to the local port you want the API to be running on
```
$ docker run -it -p 5000:5000 vlall/moses-api
```
- You will be directed to the /mosesdecoder directory. Here is where you will compile Moses (takes a few minutes to run).
```
$ ./bjam --with-boost=/home/moses/Downloads/boost_1_60_0 --with-cmph=/home/moses/cmph-2.0 --with-irstlm=/home/moses/irstlm -j12
```
- Your Docker image is already modified to work with the sample model. Use /home/moses/Downloads/moses-api/config.yaml as a reference for integration with your own models.
- Start your RESTful API server in the background
```$ python run_moses.py &```
- For a quick translation, run:
```
$ curl -XGET localhost:5000/haus | python -m json.tool
{
"CMD": "/home/moses/mosesdecoder/bin/moses -f phrase-model/moses.ini < phrase-model/in > out",
"DURATION": "0.603 seconds",
"INPUT": "haus",
"INPUT_PATH": "/home/moses/moses-models/sample-models/phrase-model/in",
"INPUT_SIZE": 4,
"LAN": "N/A",
"MODEL": "/home/moses/moses-models/sample-models",
"OUTPUT": "house",
"OUTPUT_PATH": "/home/moses/moses-models/sample-models/out",
"OUTPUT_SIZE": 7,
"STATUS": "Files successfully read",
"URL": "http://localhost:5000"
}
```
- To translate a whole file, navigate to your /moses-api folder, then run:
```
$ curl -XPUT -F name=@translate_me.txt localhost:5000/upload | python -m json.tool
{
"CMD": "/home/moses/mosesdecoder/bin/moses -f phrase-model/moses.ini < phrase-model/in > out",
"DURATION": "0.624 seconds",
"INPUT": "das ist ein klein haus\n",
"INPUT_PATH": "/home/moses/moses-models/sample-models/phrase-model/in",
"INPUT_SIZE": 23,
"LAN": "N/A",
"MODEL": "/home/moses/moses-models/sample-models",
"OUTPUT": "this is a small house",
"OUTPUT_PATH": "/home/moses/moses-models/sample-models/out",
"OUTPUT_SIZE": 24,
"STATUS": "Files successfully read",
"URL": "http://localhost:5000"
}
```
Since this the Moses sample-model, it is very limited. Visit the Moses SMT website for more information on creating your own models: http://www.statmt.org/moses/?n=Development.GetStarted
run/config.yaml 0 → 100644
sample-models:
in: '/home/tugasakhir/ta16/works/mert-work/in'
out: '/home/tugasakhir/ta16/works/out'
homeDir: '/home/tugasakhir/ta16/works'
command: '/home/tugasakhir/moses/mosesdecoder/bin/moses -f mert-work/moses.ini < mert-work/in > out'
run/moses_api.py 0 → 100644
from flask import Flask, request, url_for, render_template, jsonify
from flask_api import FlaskAPI, status, exceptions
import subprocess
import yaml
from werkzeug.utils import secure_filename
import codecs
import json
import time
app = Flask(__name__)
ALLOWED_EXTENSIONS = set(['txt', 'pdf'])
def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
def translate(text):
start_time = time.time()
"""
Run translation model using config
"""
with open("/home/tugasakhir/run/config.yaml", 'r') as f:
doc = yaml.safe_load(f)
fileIn = doc['sample-models']['in']
fileOut = doc['sample-models']['out']
homeDir = doc['sample-models']['homeDir']
runCommand = doc['sample-models']['command']
status = 'Files successfully read'
subprocess.call(['rm %s && rm %s' % (fileIn, fileOut)], shell=True)
text8 = text
inputFile = open(fileIn, 'w')
inputFile.write(text8 + '\n')
inputFile.close()
subprocess.call([runCommand], cwd=homeDir, shell=True)
readTranslate = open(fileOut, 'r')
translatedText = readTranslate.read()
readTranslate.close()
##return {
##"STATUS": status,
##"LAN": 'N/A',
##"MODEL": str(homeDir),
##"CMD": str(runCommand),
##"URL": request.host_url.rstrip('/'),
##"INPUT": text,
##"INPUT_SIZE": len(text),
##"INPUT_PATH": str(fileIn),
##"OUTPUT": translatedText.rstrip(),
##"OUTPUT_SIZE": len(translatedText),
##"OUTPUT_PATH": str(fileOut),
##"DURATION": '%.3f seconds' % (time.time() - start_time)
##}
result = {
"output": translatedText.rstrip()
}
result = {str(key): value for key, value in result.items()}
return jsonify(result=result)
@app.route('/hello/', methods=['GET', 'POST'])
def home():
return render_template('home.html')
@app.route("/", methods=['GET'])
def instructions():
return render_template('home.html')
@app.route("/<text>", methods=['GET'])
def user_get(text):
"""
Translate text
"""
text = translate(text)
return text
@app.route("/trans", methods=['GET', 'POST'])
def trans_word():
text = request.form['text']
text = translate(text)
return text
@app.route("/upload", methods=['POST', 'PUT'])
def upload():
"""
Tranlsate file
"""
file = request.files['name']
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
text = file.read()
text = translate(text)
return text
else:
return ('Error reading file...\n')
run/run_moses.py 0 → 100644
from moses_api import app
def main():
app.run(
host='0.0.0.0',
port=5003
)
if __name__ == '__main__':
main()
run/templates/home.html 0 → 100644
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Penerjemah Bahasa Gaul-Indonesia</title>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
</head>
<style type="text/css">
.shadow-textarea textarea.form-control::placeholder {
font-weight: 300;
}
.shadow-textarea textarea.form-control {
padding-left: 0.8rem;
background-color:#FFF !important;
}
textarea {
width: 200px;
box-shadow: 0 4px 4px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
}
h1{
text-align: center;
}
.container {
padding-top: 50px;
}
.textarea1 .textarea2 {
font-family: sans-serif;
}
</style>
<script>
function myFunction() {
$("#result").val('');
var text= $('#text').val();
$.ajax({
url: "/trans",
type: "POST",
data: {text:text}
}).done(function(response) {
response =response.result;
$.each(response,function(key,val){
console.log(val);
var md = $('#result')
md.val(md.val() + val)
});
});
};
</script>
<body>
<div class="container">
<h1>Penerjemah Bahasa Tidak Baku</h1>
<div class="container">
<div class="row">
<div class="col">
<div class="form-group ">
<label for="textarea1">Tidak Baku</label>
<textarea class="form-control z-depth-1" id="text" rows="3"></textarea>
</div>
</div>
<div class="col">
<div class="form-group shadow-textarea">
<label for="textarea2">Baku</label>
<textarea class="form-control z-depth-1" id="result" rows="3" placeholder="Terjemahan"></textarea>
</div>
</div>
</div>
<button class="btn btn-primary" style="margin-top: 5px;" id="clicked" onclick="myFunction()">Submit</button>
</div>
</div>
<div class="show-data" >
</div>
</body>
</html>
\ No newline at end of file
run/templates/style.css 0 → 100644
.shadow-textarea textarea.form-control::placeholder {
font-weight: 300;
}
.shadow-textarea textarea.form-control {
padding-left: 0.8rem;
background-color:#FFF !important;
}
textarea {
width: 200px;
box-shadow: 0 4px 4px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
}
h1{
text-align: center;
}
.container {
padding-top: 50px;
}
.textarea1 .textarea2 {
font-family: sans-serif;
}
run/translate_me.txt 0 → 100644
das ist ein klein haus
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment