Skip to content
Snippets Groups Projects
Commit 582f7852 authored by Duc Cao's avatar Duc Cao
Browse files

Remove wapiti Windows

parent 282f8ed1
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 49 additions and 271 deletions
lib/jar/wapiti-1.5.0-win.jar deleted 100644 → 0
+ 0
− 0
View file @ 282f8ed1
File deleted
pom.xml
+ 0
− 7
View file @ 582f7852
......@@ -118,13 +118,6 @@
<scope>system</scope>
<systemPath>${project.basedir}/lib/jar/stanford-french-corenlp-2016-01-14-models.jar</systemPath>
</dependency>
<dependency>
<groupId>fr.limsi.wapiti</groupId>
<artifactId>wapiti-win</artifactId>
<version>1.5.0</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/jar/wapiti-1.5.0-win.jar</systemPath>
</dependency>
<dependency>
<groupId>net.sf</groupId>
<artifactId>hfst</artifactId>
......
src/main/java/fr/limsi/sourceExtractor/wapiti/WapitiLabeling.java
+ 7
− 47
View file @ 582f7852
......@@ -6,17 +6,12 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ForkJoinPool;
import org.apache.commons.lang3.SystemUtils;
import fr.limsi.sourceExtractor.ProcessingThreadFactory;
import fr.limsi.wapiti.Wapiti;
public abstract class WapitiLabeling {
public abstract void label(File input, File outputDir, String filename)
public abstract void label(File input, File outputDir, String filename, File dirLib)
throws UnsupportedEncodingException, FileNotFoundException;
/**
* Méthode permettant d'utiliser wapiti sur les fichiers de tests
......@@ -41,7 +36,7 @@ public abstract class WapitiLabeling {
+ "%s "
+ "-i %s/ "
+ "-o %s/ -p", dirLib, modelFile, inputDir.getAbsolutePath(), outputDir.getAbsolutePath());
System.err.println(command);
System.out.println(command);
String line;
Process p = Runtime.getRuntime().exec(command);
......@@ -62,42 +57,13 @@ public abstract class WapitiLabeling {
e.printStackTrace();
}
}
public void wapitiTest(File input, File outputDir, int jobNumber)
throws UnsupportedEncodingException, FileNotFoundException, InterruptedException {
if (!outputDir.exists()) {
outputDir.mkdirs();
}
if (input.isDirectory()) {
ProcessingThreadFactory threadFactory = new ProcessingThreadFactory();
ForkJoinPool forkJoinPool = new ForkJoinPool(jobNumber, threadFactory, null, false);
ArrayBlockingQueue<WapitiTask> forks = new ArrayBlockingQueue<>(1000000);
for (File infile : input.listFiles()) {
if (infile.isFile()) {
WapitiTask task = new WapitiTask(infile, outputDir, this);
forks.put(task);
forkJoinPool.submit(task);
//wapitiLabelFile(infile, outputDir, this);
}
}
while (forks.size() > 0) {
WapitiTask task = forks.take();
task.join();
}
}
else {
throw new RuntimeException();
}
}
public void wapitiLabeled(File input, String fileId, String extension, File outputDir) {
public void wapitiLabeled(File input, String fileId, String extension, File outputDir, File dirLib) {
if (!outputDir.exists()) {
outputDir.mkdirs();
}
File inFile = new File(input, fileId + "." + extension);
WapitiTask task = new WapitiTask(inFile, outputDir, this);
WapitiTask task = new WapitiTask(inFile, outputDir, this, dirLib);
task.compute();
}
......@@ -116,12 +82,6 @@ public abstract class WapitiLabeling {
public static WapitiLabeling getWapitiInstance(File modelFile)
throws UnsupportedEncodingException, FileNotFoundException {
if (org.apache.commons.lang3.SystemUtils.IS_OS_LINUX || SystemUtils.IS_OS_MAC) {
return new WapitiLabelingLinux(modelFile);
} else if (SystemUtils.IS_OS_WINDOWS) {
return new WapitiLabelingWindows(modelFile);
} else {
throw new RuntimeException("Unknown OS");
}
return new WapitiLabelingLinux(modelFile);
}
}
src/main/java/fr/limsi/sourceExtractor/wapiti/WapitiLabelingLinux.java
+ 37
− 8
View file @ 582f7852
package fr.limsi.sourceExtractor.wapiti;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.limsi.wapiti.Wapiti;
public class WapitiLabelingLinux extends WapitiLabeling {
private Logger logger = LoggerFactory.getLogger(WapitiLabelingLinux.class);
......@@ -18,12 +19,40 @@ public class WapitiLabelingLinux extends WapitiLabeling {
this.modelFile = modelFile;
}
@Override
public void label(File input, File outputDir, String filename)
public void label(File input, File outputDir, String filename, File dirLib)
throws UnsupportedEncodingException, FileNotFoundException {
String wapitiArgs = "label -m " + this.modelFile+" "+input+ " "+outputDir+"/"+filename+".wapiti -p";
logger.debug(wapitiArgs);
String wapitiArgsFormat = String.format(wapitiArgs);
Wapiti.runWapiti(wapitiArgsFormat);
if (!outputDir.exists()) {
outputDir.mkdirs();
}
try {
String command = String.format("%s/wapiti"
+ " label -m "
+ "%s "
+ "-i %s "
+ "-o %s -p", dirLib.getAbsoluteFile(), this.modelFile, input.getAbsolutePath(),
outputDir.getAbsolutePath() + "/" + filename + ".wapiti");
logger.debug(command);
String line;
Process p = Runtime.getRuntime().exec(command);
BufferedReader bri = new BufferedReader(new InputStreamReader(p.getInputStream()));
BufferedReader bre = new BufferedReader(new InputStreamReader(p.getErrorStream()));
while ((line = bri.readLine()) != null) {
System.out.println(line);
}
bri.close();
while ((line = bre.readLine()) != null) {
System.out.println(line);
}
bre.close();
p.waitFor();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
src/main/java/fr/limsi/sourceExtractor/wapiti/WapitiLabelingWindows.java deleted 100644 → 0
+ 0
− 103
View file @ 282f8ed1
package fr.limsi.sourceExtractor.wapiti;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import fr.limsi.wapiti.WapitiIO;
import fr.limsi.wapiti.WapitiModel;
public class WapitiLabelingWindows extends WapitiLabeling {
private WapitiModel model;
private MyWapitiIO ioA;
public WapitiLabelingWindows(File modelsFile) throws UnsupportedEncodingException, FileNotFoundException {
this.ioA = new MyWapitiIO(
new FileInputStream(modelsFile),
null);
this.model = new WapitiModel(ioA);
}
@Override
public void finalize() {
this.ioA.close();
}
@Override
public void label(File input, File outputDir,
String filename) throws UnsupportedEncodingException, FileNotFoundException {
MyWapitiIO ioB = new MyWapitiIO(
new FileInputStream(input),
new FileOutputStream(outputDir+"/"+filename+".wapiti"));
this.model.label(ioB);
ioB.close();
}
private class MyWapitiIO extends WapitiIO {
// public int lines = 0;
// public int lines_out_count = 0;
private BufferedReader reader;
private OutputStream outputStream;
private InputStream inputStream;
public MyWapitiIO(
InputStream inputStream,
OutputStream outputStream) throws UnsupportedEncodingException
{
super();
this.reader = new BufferedReader(
new InputStreamReader(inputStream, "ISO-8859-1"));
this.outputStream = outputStream;
this.inputStream = inputStream;
}
@Override
public String readline() {
// this.lines++;
try {
return this.reader.readLine();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
@Override
public void append(String bytes) {
try {
byte[] b = bytes.getBytes("ISO-8859-1");
if (this.outputStream != null)
this.outputStream.write(b);
// this.lines_out_count++;
} catch (Exception e) {
e.printStackTrace();
}
}
public void close() {
try {
if (this.outputStream != null)
this.outputStream.close();
if (this.inputStream != null)
this.inputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
src/main/java/fr/limsi/sourceExtractor/wapiti/WapitiTask.java
+ 5
− 3
View file @ 582f7852
......@@ -14,15 +14,17 @@ public class WapitiTask extends RecursiveTask<Integer> {
private File input;
private File outputDir;
private WapitiLabeling wapitiLabeler;
private File dirLib;
public WapitiTask(File input, File outputDir, WapitiLabeling wapitiLabeler) {
public WapitiTask(File input, File outputDir, WapitiLabeling wapitiLabeler, File dirLib) {
super();
this.input = input;
this.outputDir = outputDir;
this.wapitiLabeler = wapitiLabeler;
this.dirLib = dirLib;
}
private static Integer wapitiLabelFile(File input, File outputDir, WapitiLabeling wapitiLabeler)
private Integer wapitiLabelFile(File input, File outputDir, WapitiLabeling wapitiLabeler)
throws UnsupportedEncodingException, FileNotFoundException {
String filename = input.getName();
......@@ -32,7 +34,7 @@ public class WapitiTask extends RecursiveTask<Integer> {
filename = filename.substring(0, pos);
}
wapitiLabeler.label(input, outputDir, filename);
wapitiLabeler.label(input, outputDir, filename, this.dirLib);
return 1;
}
......
src/trash/WapitiLabelingWindows.java deleted 100644 → 0
+ 0
− 103
View file @ 282f8ed1
package fr.limsi.sourceExtractor.wapiti;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import fr.limsi.wapiti.WapitiIO;
import fr.limsi.wapiti.WapitiModel;
public class WapitiLabelingWindows extends WapitiLabeling {
private WapitiModel model;
private MyWapitiIO ioA;
public WapitiLabelingWindows(File modelsFile) throws UnsupportedEncodingException, FileNotFoundException {
this.ioA = new MyWapitiIO(
new FileInputStream(modelsFile),
null);
this.model = new WapitiModel(ioA);
}
@Override
public void finalize() {
this.ioA.close();
}
@Override
public void label(File input, File outputDir,
String filename) throws UnsupportedEncodingException, FileNotFoundException {
MyWapitiIO ioB = new MyWapitiIO(
new FileInputStream(input),
new FileOutputStream(outputDir+"/"+filename+".wapiti"));
this.model.label(ioB);
ioB.close();
}
private class MyWapitiIO extends WapitiIO {
// public int lines = 0;
// public int lines_out_count = 0;
private BufferedReader reader;
private OutputStream outputStream;
private InputStream inputStream;
public MyWapitiIO(
InputStream inputStream,
OutputStream outputStream) throws UnsupportedEncodingException
{
super();
this.reader = new BufferedReader(
new InputStreamReader(inputStream, "ISO-8859-1"));
this.outputStream = outputStream;
this.inputStream = inputStream;
}
@Override
public String readline() {
// this.lines++;
try {
return this.reader.readLine();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
@Override
public void append(String bytes) {
try {
byte[] b = bytes.getBytes("ISO-8859-1");
if (this.outputStream != null)
this.outputStream.write(b);
// this.lines_out_count++;
} catch (Exception e) {
e.printStackTrace();
}
}
public void close() {
try {
if (this.outputStream != null)
this.outputStream.close();
if (this.inputStream != null)
this.inputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment