Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

replaced ZipInputStream in CreateNewDataFilesCommand #199

Open
wants to merge 20 commits into
base: v6.2-DANS-DataStation
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
try around bagItFileHandler
  • Loading branch information
jo-pol committed Sep 16, 2024
commit b93561b7bb7f1ed8045b6b572ecebef017b7c80a
11 changes: 7 additions & 4 deletions src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
Original file line number Diff line number Diff line change
@@ -515,10 +515,13 @@ public static String determineFileType(File f, String fileName) throws IOExcepti
// logger.info("------- shapefile FOUND ----------");
fileType = ShapefileHandler.SHAPEFILE_FILE_TYPE; //"application/zipped-shapefile";
}

Optional<BagItFileHandler> bagItFileHandler = BagItFileHandler.getFromCDI();
if(bagItFileHandler.isPresent() && bagItFileHandler.get().isBagItPackage(fileName, f)) {
fileType = BagItFileHandler.FILE_TYPE;
try {
Optional<BagItFileHandler> bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
if (bagItFileHandler.isPresent() && bagItFileHandler.get().isBagItPackage(fileName, f)) {
fileType = BagItFileHandler.FILE_TYPE;
}
} catch (Exception e) {
logger.warning("Error checking for BagIt package: " + e.getMessage());
}
}

Original file line number Diff line number Diff line change
@@ -11,7 +11,6 @@
import edu.harvard.iq.dataverse.util.bagit.data.FileDataProvider;
import edu.harvard.iq.dataverse.util.bagit.data.FileDataProviderFactory;
import edu.harvard.iq.dataverse.util.bagit.data.FileUtilWrapper;
import jakarta.enterprise.inject.spi.CDI;

import java.io.File;
import java.io.IOException;
@@ -45,10 +44,6 @@ public BagItFileHandler(FileUtilWrapper fileUtil, FileDataProviderFactory fileDa
this.postProcessor = postProcessor;
}

public static Optional<BagItFileHandler> getFromCDI() {
return CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
}

public boolean isBagItPackage(String uploadedFilename, File file) throws IOException {
try(FileDataProvider fileDataProvider = fileDataProviderFactory.getFileDataProvider(file)) {
boolean isBagItPackage = bagValidator.hasBagItPackage(fileDataProvider);
Original file line number Diff line number Diff line change
@@ -8,27 +8,30 @@
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
import edu.harvard.iq.dataverse.util.FileUtil;
import edu.harvard.iq.dataverse.util.JhoveFileType;
import edu.harvard.iq.dataverse.util.SystemConfig;
import edu.harvard.iq.dataverse.util.file.BagItFileHandler;
import edu.harvard.iq.dataverse.util.testing.JvmSetting;
import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

@@ -44,13 +47,19 @@
public class CreateNewDataFilesTest {
// TODO keep constants for annotations in sync with class name
Path testDir = Path.of("target/test/").resolve(getClass().getSimpleName());
PrintStream original_stderr;

@BeforeEach
public void cleanTmpDir() throws IOException {
original_stderr = System.err;
if(testDir.toFile().exists())
deleteDirectory(testDir);
}

@AfterEach void restoreStderr() {
System.setErr(original_stderr);
}

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
public void execute_fails_to_upload_when_tmp_does_not_exist() throws FileNotFoundException {
@@ -69,48 +78,18 @@ public void execute_fails_to_upload_when_tmp_does_not_exist() throws FileNotFoun

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
public void execute_fails_to_upload_too_big_files() throws FileNotFoundException {
createDirectories(Path.of("target/test/CreateNewDataFilesTest/tmp/temp"));

mockTmpLookup();
var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion());
var ctxt = mockCommandContext(mockSysConfig(true, 1000L, MD5, 10));

assertThatThrownBy(() -> cmd.execute(ctxt))
.isInstanceOf(CommandException.class)
.hasMessage("This file size (56.0 KB) exceeds the size limit of 1000 B.");
}

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
public void execute_fails_on_too_little_remaining_storage() throws FileNotFoundException {
createDirectories(Path.of("target/test/CreateNewDataFilesTest/tmp/temp"));

mockTmpLookup();
var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion());
var ctxt = mockCommandContext(mockSysConfig(true, 1000000L, MD5, 0));
try (MockedStatic<JhoveFileType> mockedStatic = Mockito.mockStatic(JhoveFileType.class)) {
mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");

assertThatThrownBy(() -> cmd.execute(ctxt))
.isInstanceOf(CommandException.class)
.hasMessage("This file (size 56.0 KB) exceeds the remaining storage quota of 500 B.");
}
}
@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
public void execute_fails_on_too_little_remaining_storage_for_unzipped_files() throws FileNotFoundException {
public void execute_fails_on_size_limit() throws Exception {
createDirectories(Path.of("target/test/CreateNewDataFilesTest/tmp/temp"));

mockTmpLookup();
var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion());
var ctxt = mockCommandContext(mockSysConfig(true, 1000000L, MD5, 10));
try (MockedStatic<JhoveFileType> mockedStatic = Mockito.mockStatic(JhoveFileType.class)) {
var cmd = createCmd("scripts/search/data/binary/3files.zip", mockDatasetVersion());
var ctxt = mockCommandContext(mockSysConfig(true, 50L, MD5, 0));
try (var mockedStatic = Mockito.mockStatic(JhoveFileType.class)) {
mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");

assertThatThrownBy(() -> cmd.execute(ctxt))
.isInstanceOf(CommandException.class)
.hasMessage("Unzipped files exceed the remaining storage quota of 500 B.");
.hasMessage("This file size (462 B) exceeds the size limit of 50 B.");
}
}

@@ -144,7 +123,7 @@ public void execute_without_shape_files() throws Exception {

@Test
@JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
public void execute_with_2_shapes() throws Exception {
public void execute_with_2_shapes_does_not_check_zipUploadFilesLimit() throws Exception {
var tempDir = testDir.resolve("tmp/temp");
createDirectories(tempDir);

@@ -158,11 +137,7 @@ public void execute_with_2_shapes() throws Exception {
mockTmpLookup();
var cmd = createCmd(testFile.toString(), mockDatasetVersion());
var ctxt = mockCommandContext(mockSysConfig(false, 1000000L, MD5, 0));
try (var mockedHandler = Mockito.mockStatic(BagItFileHandler.class);
var mockedJHoveFileType = Mockito.mockStatic(JhoveFileType.class)
) {
var opt = mockOptional(false);
mockedHandler.when(BagItFileHandler::getFromCDI).thenReturn(opt);
try (var mockedJHoveFileType = Mockito.mockStatic(JhoveFileType.class)) {
mockedJHoveFileType.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");

// the test
@@ -185,14 +160,11 @@ public void execute_with_2_shapes() throws Exception {
}
}

private @NotNull Optional mockOptional(boolean t) {
var mockedOptional = Mockito.mock(Optional.class);
Mockito.when(mockedOptional.isPresent()).thenReturn(t);
return mockedOptional;
}

// simplified version from ShapefileHandlerTest
private File createAndZipFiles(List<String> file_names, Path zipfile) throws IOException {
// TODO provoke java.util.zip.ZipException: only DEFLATED entries can have EXT descriptor
// and test target/test/CreateNewDataFilesTest/shapes.zip on VM/demo
// write a test that tries to read it with ZipInputStream
try (ZipOutputStream zip_stream = new ZipOutputStream(new FileOutputStream(zipfile.toFile()))) {
for (var file_name : file_names) {
zip_stream.putNextEntry(new ZipEntry(file_name));