Commit 582f7852 authored by Duc Cao's avatar Duc Cao

Remove wapiti Windows

parent 282f8ed1
......@@ -118,13 +118,6 @@
<scope>system</scope>
<systemPath>${project.basedir}/lib/jar/stanford-french-corenlp-2016-01-14-models.jar</systemPath>
</dependency>
<dependency>
<groupId>fr.limsi.wapiti</groupId>
<artifactId>wapiti-win</artifactId>
<version>1.5.0</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/jar/wapiti-1.5.0-win.jar</systemPath>
</dependency>
<dependency>
<groupId>net.sf</groupId>
<artifactId>hfst</artifactId>
......
......@@ -6,17 +6,12 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ForkJoinPool;
import org.apache.commons.lang3.SystemUtils;
import fr.limsi.sourceExtractor.ProcessingThreadFactory;
import fr.limsi.wapiti.Wapiti;
public abstract class WapitiLabeling {
public abstract void label(File input, File outputDir, String filename)
public abstract void label(File input, File outputDir, String filename, File dirLib)
throws UnsupportedEncodingException, FileNotFoundException;
/**
* Méthode permettant d'utiliser wapiti sur les fichiers de tests
......@@ -41,7 +36,7 @@ public abstract class WapitiLabeling {
+ "%s "
+ "-i %s/ "
+ "-o %s/ -p", dirLib, modelFile, inputDir.getAbsolutePath(), outputDir.getAbsolutePath());
System.err.println(command);
System.out.println(command);
String line;
Process p = Runtime.getRuntime().exec(command);
......@@ -62,42 +57,13 @@ public abstract class WapitiLabeling {
e.printStackTrace();
}
}
public void wapitiTest(File input, File outputDir, int jobNumber)
throws UnsupportedEncodingException, FileNotFoundException, InterruptedException {
if (!outputDir.exists()) {
outputDir.mkdirs();
}
if (input.isDirectory()) {
ProcessingThreadFactory threadFactory = new ProcessingThreadFactory();
ForkJoinPool forkJoinPool = new ForkJoinPool(jobNumber, threadFactory, null, false);
ArrayBlockingQueue<WapitiTask> forks = new ArrayBlockingQueue<>(1000000);
for (File infile : input.listFiles()) {
if (infile.isFile()) {
WapitiTask task = new WapitiTask(infile, outputDir, this);
forks.put(task);
forkJoinPool.submit(task);
//wapitiLabelFile(infile, outputDir, this);
}
}
while (forks.size() > 0) {
WapitiTask task = forks.take();
task.join();
}
}
else {
throw new RuntimeException();
}
}
public void wapitiLabeled(File input, String fileId, String extension, File outputDir) {
public void wapitiLabeled(File input, String fileId, String extension, File outputDir, File dirLib) {
if (!outputDir.exists()) {
outputDir.mkdirs();
}
File inFile = new File(input, fileId + "." + extension);
WapitiTask task = new WapitiTask(inFile, outputDir, this);
WapitiTask task = new WapitiTask(inFile, outputDir, this, dirLib);
task.compute();
}
......@@ -116,12 +82,6 @@ public abstract class WapitiLabeling {
public static WapitiLabeling getWapitiInstance(File modelFile)
throws UnsupportedEncodingException, FileNotFoundException {
if (org.apache.commons.lang3.SystemUtils.IS_OS_LINUX || SystemUtils.IS_OS_MAC) {
return new WapitiLabelingLinux(modelFile);
} else if (SystemUtils.IS_OS_WINDOWS) {
return new WapitiLabelingWindows(modelFile);
} else {
throw new RuntimeException("Unknown OS");
}
return new WapitiLabelingLinux(modelFile);
}
}
package fr.limsi.sourceExtractor.wapiti;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.limsi.wapiti.Wapiti;
public class WapitiLabelingLinux extends WapitiLabeling {
private Logger logger = LoggerFactory.getLogger(WapitiLabelingLinux.class);
......@@ -18,12 +19,40 @@ public class WapitiLabelingLinux extends WapitiLabeling {
this.modelFile = modelFile;
}
@Override
public void label(File input, File outputDir, String filename)
public void label(File input, File outputDir, String filename, File dirLib)
throws UnsupportedEncodingException, FileNotFoundException {
String wapitiArgs = "label -m " + this.modelFile+" "+input+ " "+outputDir+"/"+filename+".wapiti -p";
logger.debug(wapitiArgs);
String wapitiArgsFormat = String.format(wapitiArgs);
Wapiti.runWapiti(wapitiArgsFormat);
if (!outputDir.exists()) {
outputDir.mkdirs();
}
try {
String command = String.format("%s/wapiti"
+ " label -m "
+ "%s "
+ "-i %s "
+ "-o %s -p", dirLib.getAbsoluteFile(), this.modelFile, input.getAbsolutePath(),
outputDir.getAbsolutePath() + "/" + filename + ".wapiti");
logger.debug(command);
String line;
Process p = Runtime.getRuntime().exec(command);
BufferedReader bri = new BufferedReader(new InputStreamReader(p.getInputStream()));
BufferedReader bre = new BufferedReader(new InputStreamReader(p.getErrorStream()));
while ((line = bri.readLine()) != null) {
System.out.println(line);
}
bri.close();
while ((line = bre.readLine()) != null) {
System.out.println(line);
}
bre.close();
p.waitFor();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
package fr.limsi.sourceExtractor.wapiti;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import fr.limsi.wapiti.WapitiIO;
import fr.limsi.wapiti.WapitiModel;
public class WapitiLabelingWindows extends WapitiLabeling {
private WapitiModel model;
private MyWapitiIO ioA;
public WapitiLabelingWindows(File modelsFile) throws UnsupportedEncodingException, FileNotFoundException {
this.ioA = new MyWapitiIO(
new FileInputStream(modelsFile),
null);
this.model = new WapitiModel(ioA);
}
@Override
public void finalize() {
this.ioA.close();
}
@Override
public void label(File input, File outputDir,
String filename) throws UnsupportedEncodingException, FileNotFoundException {
MyWapitiIO ioB = new MyWapitiIO(
new FileInputStream(input),
new FileOutputStream(outputDir+"/"+filename+".wapiti"));
this.model.label(ioB);
ioB.close();
}
private class MyWapitiIO extends WapitiIO {
// public int lines = 0;
// public int lines_out_count = 0;
private BufferedReader reader;
private OutputStream outputStream;
private InputStream inputStream;
public MyWapitiIO(
InputStream inputStream,
OutputStream outputStream) throws UnsupportedEncodingException
{
super();
this.reader = new BufferedReader(
new InputStreamReader(inputStream, "ISO-8859-1"));
this.outputStream = outputStream;
this.inputStream = inputStream;
}
@Override
public String readline() {
// this.lines++;
try {
return this.reader.readLine();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
@Override
public void append(String bytes) {
try {
byte[] b = bytes.getBytes("ISO-8859-1");
if (this.outputStream != null)
this.outputStream.write(b);
// this.lines_out_count++;
} catch (Exception e) {
e.printStackTrace();
}
}
public void close() {
try {
if (this.outputStream != null)
this.outputStream.close();
if (this.inputStream != null)
this.inputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
......@@ -14,15 +14,17 @@ public class WapitiTask extends RecursiveTask<Integer> {
private File input;
private File outputDir;
private WapitiLabeling wapitiLabeler;
private File dirLib;
public WapitiTask(File input, File outputDir, WapitiLabeling wapitiLabeler) {
public WapitiTask(File input, File outputDir, WapitiLabeling wapitiLabeler, File dirLib) {
super();
this.input = input;
this.outputDir = outputDir;
this.wapitiLabeler = wapitiLabeler;
this.dirLib = dirLib;
}
private static Integer wapitiLabelFile(File input, File outputDir, WapitiLabeling wapitiLabeler)
private Integer wapitiLabelFile(File input, File outputDir, WapitiLabeling wapitiLabeler)
throws UnsupportedEncodingException, FileNotFoundException {
String filename = input.getName();
......@@ -32,7 +34,7 @@ public class WapitiTask extends RecursiveTask<Integer> {
filename = filename.substring(0, pos);
}
wapitiLabeler.label(input, outputDir, filename);
wapitiLabeler.label(input, outputDir, filename, this.dirLib);
return 1;
}
......
package fr.limsi.sourceExtractor.wapiti;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import fr.limsi.wapiti.WapitiIO;
import fr.limsi.wapiti.WapitiModel;
public class WapitiLabelingWindows extends WapitiLabeling {
private WapitiModel model;
private MyWapitiIO ioA;
public WapitiLabelingWindows(File modelsFile) throws UnsupportedEncodingException, FileNotFoundException {
this.ioA = new MyWapitiIO(
new FileInputStream(modelsFile),
null);
this.model = new WapitiModel(ioA);
}
@Override
public void finalize() {
this.ioA.close();
}
@Override
public void label(File input, File outputDir,
String filename) throws UnsupportedEncodingException, FileNotFoundException {
MyWapitiIO ioB = new MyWapitiIO(
new FileInputStream(input),
new FileOutputStream(outputDir+"/"+filename+".wapiti"));
this.model.label(ioB);
ioB.close();
}
private class MyWapitiIO extends WapitiIO {
// public int lines = 0;
// public int lines_out_count = 0;
private BufferedReader reader;
private OutputStream outputStream;
private InputStream inputStream;
public MyWapitiIO(
InputStream inputStream,
OutputStream outputStream) throws UnsupportedEncodingException
{
super();
this.reader = new BufferedReader(
new InputStreamReader(inputStream, "ISO-8859-1"));
this.outputStream = outputStream;
this.inputStream = inputStream;
}
@Override
public String readline() {
// this.lines++;
try {
return this.reader.readLine();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
@Override
public void append(String bytes) {
try {
byte[] b = bytes.getBytes("ISO-8859-1");
if (this.outputStream != null)
this.outputStream.write(b);
// this.lines_out_count++;
} catch (Exception e) {
e.printStackTrace();
}
}
public void close() {
try {
if (this.outputStream != null)
this.outputStream.close();
if (this.inputStream != null)
this.inputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment