Commit 5887cba1 authored by Bertrand Goupil's avatar Bertrand Goupil

Bug fix: clean Memory object to avoid memory leak.

Add support to JAVA_OPT into Docker instances.
parent f9a72390
...@@ -20,5 +20,6 @@ ADD limsiSourceExtractor.jar limsiSourceExtractor.jar ...@@ -20,5 +20,6 @@ ADD limsiSourceExtractor.jar limsiSourceExtractor.jar
EXPOSE 8080 EXPOSE 8080
RUN sh -c 'touch /limsiSourceExtractor.jar' RUN sh -c 'touch /limsiSourceExtractor.jar'
ENV JAVA_OPTS="-Dspring.config.location=/configuration/" #ENV JAVA_OPTS="-Dspring.config.location=/configuration/"
ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /limsiSourceExtractor.jar" ] #ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Dspring.config.location=/configuration/ -Djava.security.egd=file:/dev/./urandom -jar /limsiSourceExtractor.jar" ]
\ No newline at end of file ENTRYPOINT exec java $JAVA_OPTS -Dspring.config.location=/configuration/ -Djava.security.egd=file:/dev/./urandom -jar /limsiSourceExtractor.jar
\ No newline at end of file
...@@ -15,5 +15,11 @@ public class Memory { ...@@ -15,5 +15,11 @@ public class Memory {
this.docTexts = docTexts; this.docTexts = docTexts;
this.sentenceOffsetsByFile = sentenceOffsetsByFile; this.sentenceOffsetsByFile = sentenceOffsetsByFile;
} }
public void cleanEntry(String fileId) {
this.parsedTexts.remove(fileId);
this.docTexts.remove(fileId);
this.sentenceOffsetsByFile.remove(fileId);
}
} }
...@@ -10,6 +10,7 @@ import org.slf4j.Logger; ...@@ -10,6 +10,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import fr.limsi.sourceExtractor.DIRUtils; import fr.limsi.sourceExtractor.DIRUtils;
import fr.limsi.sourceExtractor.Memory;
import fr.limsi.sourceExtractor.Paths; import fr.limsi.sourceExtractor.Paths;
import fr.limsi.sourceExtractor.application.configuration.SourceExtractorConfig; import fr.limsi.sourceExtractor.application.configuration.SourceExtractorConfig;
import fr.limsi.sourceExtractor.application.configuration.SourceExtractorConstant; import fr.limsi.sourceExtractor.application.configuration.SourceExtractorConstant;
...@@ -66,6 +67,8 @@ public class SimpleExtractSource extends TrainingUnLabel { ...@@ -66,6 +67,8 @@ public class SimpleExtractSource extends TrainingUnLabel {
String response = parseFileToAddAnnotationToBrat(fileId, annContent); String response = parseFileToAddAnnotationToBrat(fileId, annContent);
logger.debug("output: {}", response); logger.debug("output: {}", response);
cleanDirContent(this.paths.DATA_UNLABELED_DIR, fileId); cleanDirContent(this.paths.DATA_UNLABELED_DIR, fileId);
//Remote Memory entries
extractorConfig.getMemory().cleanEntry(fileId);
return response; return response;
} }
......
...@@ -29,6 +29,7 @@ import org.apache.commons.collections4.CollectionUtils; ...@@ -29,6 +29,7 @@ import org.apache.commons.collections4.CollectionUtils;
import com.google.common.base.Charsets; import com.google.common.base.Charsets;
import com.google.common.io.Files; import com.google.common.io.Files;
import edu.stanford.nlp.pipeline.Annotation;
import fr.limsi.sourceExtractor.DIRUtils; import fr.limsi.sourceExtractor.DIRUtils;
import fr.limsi.sourceExtractor.Memory; import fr.limsi.sourceExtractor.Memory;
import fr.limsi.sourceExtractor.Pair; import fr.limsi.sourceExtractor.Pair;
...@@ -159,7 +160,9 @@ public abstract class AProcess extends AProcessSupport { ...@@ -159,7 +160,9 @@ public abstract class AProcess extends AProcessSupport {
//inputTxtFile = new File(this.paths.DIR_INPUT_UNLABELED, fileId + ".txt"); //inputTxtFile = new File(this.paths.DIR_INPUT_UNLABELED, fileId + ".txt");
} }
//le fichier n'est jamais utilisé => remplacement par null (à tester possible effet de bord) //le fichier n'est jamais utilisé => remplacement par null (à tester possible effet de bord)
taggedTextOnlyOffset = parseText(fileId, SourceExtractorUtil.getDocText(fileId, null, memory), tools, resources, memory); String text = SourceExtractorUtil.getDocText(fileId, null, memory);
Annotation annotation = new Annotation(text);
taggedTextOnlyOffset = parseText(annotation, fileId, text, tools, resources, memory);
HashMap<String, String> retrieveOffsetsOfSource = new HashMap<>(); HashMap<String, String> retrieveOffsetsOfSource = new HashMap<>();
...@@ -446,8 +449,8 @@ public abstract class AProcess extends AProcessSupport { ...@@ -446,8 +449,8 @@ public abstract class AProcess extends AProcessSupport {
text = SourceExtractorUtil.getDocText(fileId, new File(paths.DIR_INPUT_UNLABELED, fileId + ".txt"), memory); text = SourceExtractorUtil.getDocText(fileId, new File(paths.DIR_INPUT_UNLABELED, fileId + ".txt"), memory);
} }
Annotation annotation = new Annotation(text);
String tagWithOffset = parseText(fileId, text, tools, resources, memory); String tagWithOffset = parseText(annotation, fileId, text, tools, resources, memory);
String resultWapiti = ""; String resultWapiti = "";
String line; String line;
......
...@@ -80,10 +80,11 @@ public abstract class AProcessSupport { ...@@ -80,10 +80,11 @@ public abstract class AProcessSupport {
public String tagAndConvert(String fileId, String text, File outFileSEC, String ann, Tools tools, Resources resources, Memory memory) public String tagAndConvert(String fileId, String text, File outFileSEC, String ann, Tools tools, Resources resources, Memory memory)
throws IOException { throws IOException {
// we use malt parser to keep the subjects found // we use malt parser to keep the subjects found
String maltParsertext = parseTextForMaltParser(text, tools, resources); Annotation document = new Annotation(text);
String maltParsertext = parseTextForMaltParser(document, text, tools, resources);
// we parse the text to tag each word of this text. // we parse the text to tag each word of this text.
String parsedText = parseText(fileId, text, tools, resources, memory); String parsedText = parseText(document, fileId, text, tools, resources, memory);
// we join both parse text // we join both parse text
String taggedText = tagFromMaltParserAndParseText(parsedText, maltParsertext, resources); String taggedText = tagFromMaltParserAndParseText(parsedText, maltParsertext, resources);
...@@ -126,14 +127,14 @@ public abstract class AProcessSupport { ...@@ -126,14 +127,14 @@ public abstract class AProcessSupport {
* Le text à tag. * Le text à tag.
* @return Le StringBuilder créé avec toutes les informations nécessaires. (6 champs) * @return Le StringBuilder créé avec toutes les informations nécessaires. (6 champs)
*/ */
protected String parseText(String fileId, String text, Tools tools, Resources resources, Memory memory) { protected String parseText(Annotation document, String fileId, String text, Tools tools, Resources resources, Memory memory) {
String parsedText = memory.parsedTexts.get(fileId); String parsedText = memory.parsedTexts.get(fileId);
if (parsedText != null) { if (parsedText != null) {
return parsedText; return parsedText;
} }
// create an empty Annotation just with the given text // create an empty Annotation just with the given text
Annotation document = new Annotation(text); //Annotation document = new Annotation(text);
// run all Annotators on this text // run all Annotators on this text
tools.getCoreNLPParser().annotate(document); tools.getCoreNLPParser().annotate(document);
...@@ -516,9 +517,9 @@ public abstract class AProcessSupport { ...@@ -516,9 +517,9 @@ public abstract class AProcessSupport {
* @return * @return
* @throws IOException * @throws IOException
*/ */
private String parseTextForMaltParser(String text, Tools tools, Resources resources) throws IOException { private String parseTextForMaltParser(Annotation document, String text, Tools tools, Resources resources) throws IOException {
// create an empty Annotation just with the given text // create an empty Annotation just with the given text
Annotation document = new Annotation(text); //Annotation document = new Annotation(text);
// run all Annotators on this text // run all Annotators on this text
tools.coreNLPParser.annotate(document); tools.coreNLPParser.annotate(document);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment