下面列出了javax.ws.rs.core.MediaType#MULTIPART_FORM_DATA_TYPE 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private void uploadHomeFile() throws Exception {
final FormDataMultiPart form = new FormDataMultiPart();
final FormDataBodyPart fileBody = new FormDataBodyPart("file", com.dremio.common.util.FileUtils.getResourceAsString("/testfiles/yelp_biz.json"), MediaType.MULTIPART_FORM_DATA_TYPE);
form.bodyPart(fileBody);
form.bodyPart(new FormDataBodyPart("fileName", "biz"));
com.dremio.file.File file1 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_start/").queryParam("extension", "json"))
.buildPost(Entity.entity(form, form.getMediaType())), com.dremio.file.File.class);
file1 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_finish/biz"))
.buildPost(Entity.json(file1.getFileFormat().getFileFormat())), com.dremio.file.File.class);
homeFileId = file1.getId();
FileFormat fileFormat = file1.getFileFormat().getFileFormat();
assertEquals(physicalFileAtHome.getLeaf().getName(), fileFormat.getName());
assertEquals(physicalFileAtHome.toPathList(), fileFormat.getFullPath());
assertEquals(FileType.JSON, fileFormat.getFileType());
fileBody.cleanup();
}
@Test // DX-5410
public void formatChangeForUploadedHomeFile() throws Exception {
FormDataMultiPart form = new FormDataMultiPart();
FormDataBodyPart fileBody = new FormDataBodyPart("file", FileUtils.getResourceAsFile("/datasets/csv/pipe.csv"), MediaType.MULTIPART_FORM_DATA_TYPE);
form.bodyPart(fileBody);
form.bodyPart(new FormDataBodyPart("fileName", "pipe"));
doc("uploading a text file");
File file1 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_start/").queryParam("extension", "csv"))
.buildPost(Entity.entity(form, form.getMediaType())), File.class);
file1 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_finish/pipe"))
.buildPost(Entity.json(file1.getFileFormat().getFileFormat())), File.class);
final FileFormat defaultFileFormat = file1.getFileFormat().getFileFormat();
assertTrue(defaultFileFormat instanceof TextFileConfig);
assertEquals(",", ((TextFileConfig)defaultFileFormat).getFieldDelimiter());
doc("change the format settings of uploaded file");
final TextFileConfig newFileFormat = (TextFileConfig)defaultFileFormat;
newFileFormat.setFieldDelimiter("|");
final FileFormat updatedFileFormat = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/file_format/pipe"))
.buildPut(Entity.json(newFileFormat)), FileFormatUI.class).getFileFormat();
assertTrue(updatedFileFormat instanceof TextFileConfig);
assertEquals("|", ((TextFileConfig)updatedFileFormat).getFieldDelimiter());
}
@Test
public void testUploadDisabled() throws Exception {
try {
// disable uploads
getSabotContext().getOptionManager().setOption(OptionValue.createBoolean(OptionValue.OptionType.SYSTEM, UIOptions.ALLOW_FILE_UPLOADS.getOptionName(), false));
FormDataMultiPart form = new FormDataMultiPart();
FormDataBodyPart fileBody = new FormDataBodyPart("file", FileUtils.getResourceAsFile("/datasets/csv/pipe.csv"), MediaType.MULTIPART_FORM_DATA_TYPE);
form.bodyPart(fileBody);
form.bodyPart(new FormDataBodyPart("fileName", "pipe"));
expectStatus(Response.Status.FORBIDDEN, getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_start/").queryParam("extension", "csv")).buildPost(Entity.entity(form, form.getMediaType())));
} finally {
// re-enable uploads
getSabotContext().getOptionManager().setOption(OptionValue.createBoolean(OptionValue.OptionType.SYSTEM, UIOptions.ALLOW_FILE_UPLOADS.getOptionName(), true));
}
}
protected final Response getFormPostResponse(String data,
String endpoint,
Class<? extends OutputStream> compressingClass,
String encoding) throws IOException {
byte[] bytes;
if (compressingClass == null) {
bytes = data.getBytes(StandardCharsets.UTF_8);
} else {
bytes = compress(data, compressingClass);
}
MediaType type =
encoding == null ? MediaType.TEXT_PLAIN_TYPE : new MediaType("application", encoding);
InputStream in = new ByteArrayInputStream(bytes);
StreamDataBodyPart filePart = new StreamDataBodyPart("data", in, "data", type);
try (MultiPart multiPart = new MultiPart(MediaType.MULTIPART_FORM_DATA_TYPE)) {
multiPart.getBodyParts().add(filePart);
return target(endpoint).request().post(
Entity.entity(multiPart, MediaType.MULTIPART_FORM_DATA_TYPE));
}
}
private Response upload(String endpoint, Iterable<String> targets, String path, String fileName, byte[] content) throws IOException {
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content)) {
StreamDataBodyPart streamDataBodyPart = new StreamDataBodyPart("file", inputStream, fileName);
try (FormDataMultiPart multiPart = new FormDataMultiPart()) {
try (FormDataMultiPart pathField = multiPart.field("path", path)) {
try (FormDataMultiPart targetsField = pathField.field("targets", String.join(",", targets))) {
try (MultiPart bodyPart = targetsField.bodyPart(streamDataBodyPart)) {
MediaType contentType = MediaType.MULTIPART_FORM_DATA_TYPE;
contentType = Boundary.addBoundary(contentType);
String signature = PkiUtil.generateSignature(signatureKey, content);
return saltTarget.path(endpoint).request().header(SIGN_HEADER, signature).post(Entity.entity(bodyPart, contentType));
}
}
}
}
}
}
@Test
public void writesCollectionOfOutputItemsAndGenerateBoundary() throws Exception {
OutputItem item1 = anOutputItem().withName("item1").withEntity("item1 entity").withMediaType(TEXT_PLAIN_TYPE).withFilename("item1.txt").build();
OutputItem item2 = anOutputItem().withName("item2").withEntity("{\"item2\":\"entity\"}").withMediaType(APPLICATION_JSON_TYPE).withFilename("item2.json").build();
Collection<OutputItem> items = newArrayList(item1, item2);
Class<Collection> type = Collection.class;
ParameterizedType genericType = ParameterizedTypeImpl.newParameterizedType(Collection.class, OutputItem.class);
Annotation[] annotations = new Annotation[0];
MediaType mediaType = MediaType.MULTIPART_FORM_DATA_TYPE;
MultivaluedMap<String, Object> headers = new MultivaluedHashMap<>();
OutputStream out = new ByteArrayOutputStream();
collectionMultipartFormDataMessageBodyWriter.writeTo(items, type, genericType, annotations, mediaType, headers, out);
MediaType mediaTypeWithBoundary = (MediaType)headers.getFirst("Content-type");
assertNotNull(mediaTypeWithBoundary);
String boundary = mediaTypeWithBoundary.getParameters().get("boundary");
assertNotNull(boundary);
verify(multipartFormDataWriter).writeItems(same(items), same(out), aryEq(boundary.getBytes()));
}
public void shouldUploadApiMedia() {
StreamDataBodyPart filePart = new StreamDataBodyPart("file",
this.getClass().getResourceAsStream("/media/logo.svg"), "logo.svg", MediaType.valueOf("image/svg+xml"));
final MultiPart multiPart = new MultiPart(MediaType.MULTIPART_FORM_DATA_TYPE);
multiPart.bodyPart(filePart);
final Response response = target(API + "/media/upload").request().post(entity(multiPart, multiPart.getMediaType()));
assertEquals(OK_200, response.getStatus());
}
public void shouldNotUploadApiMediaBecauseWrongMediaType() {
StreamDataBodyPart filePart = new StreamDataBodyPart("file",
this.getClass().getResourceAsStream("/media/logo.svg"), "logo.svg", MediaType.APPLICATION_OCTET_STREAM_TYPE);
final MultiPart multiPart = new MultiPart(MediaType.MULTIPART_FORM_DATA_TYPE);
multiPart.bodyPart(filePart);
final Response response = target(API + "/media/upload").request().post(entity(multiPart, multiPart.getMediaType()));
assertEquals(BAD_REQUEST_400, response.getStatus());
final String message = response.readEntity(String.class);
assertTrue(message, message.contains("File format unauthorized"));
}
private void shouldNotUpdateApiMediaBecauseXSS(final String fileName) {
final StreamDataBodyPart filePart = new StreamDataBodyPart("file",
this.getClass().getResourceAsStream("/media/" + fileName), fileName, IMAGE_SVG_XML_TYPE);
final MultiPart multiPart = new MultiPart(MediaType.MULTIPART_FORM_DATA_TYPE);
multiPart.bodyPart(filePart);
final Response response = target(API + "/media/upload").request().post(entity(multiPart, multiPart.getMediaType()));
assertEquals(BAD_REQUEST_400, response.getStatus());
final String message = response.readEntity(String.class);
assertTrue(message, message.contains("Invalid image format"));
}
public void sendFileToRemoteAndUnZip(final List<String> filesPath, final String saveToPath) {
Response response = null;
final MultiPart multiPart = new MultiPart(MediaType.MULTIPART_FORM_DATA_TYPE);
filesPath.stream().forEach(path -> multiPart.bodyPart(new FileDataBodyPart("file",
new File(separatorsToSystem(path)), MediaType.APPLICATION_OCTET_STREAM_TYPE)));
try {
response = client.target("http://" + ip + ":" + port + "/selenium")
.path("content")
.path("upload")
.queryParam("saveTo", separatorsToSystem(saveToPath))
.queryParam("unZip", true)
.request(MediaType.APPLICATION_JSON_TYPE)
.post(Entity.entity(multiPart, multiPart.getMediaType()));
final int status = response.getStatus();
if (status == Response.Status.OK.getStatusCode()) {
CLIENT_LOGGER.info("File(-s) " + filesPath + " has been saved to " + separatorsToSystem(saveToPath) + " on " + ip);
} else {
CLIENT_LOGGER.severe("Unable to save or unZip file(-s) " + filesPath + " to " + separatorsToSystem(saveToPath) + " on " + ip + "; status = " + status);
}
} catch (Exception e) {
CLIENT_LOGGER.info("An error occurred while file(-s) uploading: " + e);
response = null;
} finally {
if (response != null) {
response.close();
}
}
}
private void testUploadExcelFile(final boolean isXLS) throws Exception {
final String extension = isXLS ? "xls" : "xlsx";
final FileType fileType = isXLS ? FileType.XLS : FileType.EXCEL;
FormDataMultiPart form = new FormDataMultiPart();
FormDataBodyPart fileBody = new FormDataBodyPart("file", FileUtils.getResourceAsFile("/testfiles/excel." + extension), MediaType.MULTIPART_FORM_DATA_TYPE);
form.bodyPart(fileBody);
form.bodyPart(new FormDataBodyPart("fileName", "excel"));
doc("uploading excel file");
File file1 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_start/")
.queryParam("extension", extension))
.buildPost(Entity.entity(form, form.getMediaType())), File.class);
file1 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_finish/excel"))
.buildPost(Entity.json(file1.getFileFormat().getFileFormat())), File.class);
FileFormat file1Format = file1.getFileFormat().getFileFormat();
assertEquals("excel", file1Format.getName());
assertEquals(asList(HOME_NAME, "excel"), file1Format.getFullPath());
assertEquals(fileType, file1Format.getFileType());
fileBody.cleanup();
doc("getting a excel file");
File file2 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/file/excel")).buildGet(), File.class);
FileFormat file2Format = file2.getFileFormat().getFileFormat();
assertEquals("excel", file2Format.getName());
assertEquals(asList(HOME_NAME, "excel"), file2Format.getFullPath());
assertEquals(fileType, file2Format.getFileType());
doc("querying excel file");
try (final JobDataFragment truncData = submitJobAndGetData(l(JobsService.class), JobRequest.newBuilder().setSqlQuery(new SqlQuery("select * from \"" + HOME_NAME + "\".\"excel\"", SampleDataPopulator.DEFAULT_USER_NAME)).build(),
0, 500, allocator)) {
assertEquals(6, truncData.getReturnedRowCount());
assertEquals(5, truncData.getColumns().size());
}
doc("previewing excel file");
if (file2Format instanceof ExcelFileConfig) {
((ExcelFileConfig) file2Format).setExtractHeader(true);
} else {
((XlsFileConfig) file2Format).setExtractHeader(true);
}
JobDataFragment data = expectSuccess(getBuilder(getAPIv2().path("/home/" + HOME_NAME + "/file_preview/excel")).buildPost(Entity.json(file2Format)), JobDataFragment.class);
assertEquals(5, data.getReturnedRowCount());
assertEquals(5, data.getColumns().size());
doc("creating dataset from excel file");
InitialPreviewResponse previewResponse = expectSuccess(getBuilder(getAPIv2().path(
"/home/" + HOME_NAME + "/new_untitled_from_file/excel")).buildPost(Entity.json("")), InitialPreviewResponse.class);
assertEquals(5, previewResponse.getData().getColumns().size());
}
@Test
public void testHomeUploadValidation() throws Exception {
Home home = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME)).buildGet(), Home.class);
assertNotNull(home.getId());
String homeFileName = "file2";
java.io.File inputFile = temporaryFolder.newFile("input-2.json");
try (FileWriter fileWriter = new FileWriter(inputFile)) {
fileWriter.write("{\"person_id\": 1, \"salary\": 10}");
}
FormDataMultiPart form = new FormDataMultiPart();
FormDataBodyPart fileBody = new FormDataBodyPart("file", inputFile, MediaType.MULTIPART_FORM_DATA_TYPE);
form.bodyPart(fileBody);
FormDataBodyPart fileNameBody = new FormDataBodyPart("fileName", homeFileName);
form.bodyPart(fileNameBody);
doc("upload file to staging");
File file1Staged = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_start/").queryParam("extension", "json")).buildPost(
Entity.entity(form, form.getMediaType())), File.class);
FileFormat file1StagedFormat = file1Staged.getFileFormat().getFileFormat();
assertEquals(homeFileName, file1StagedFormat.getName());
assertEquals(asList(HOME_NAME, homeFileName), file1StagedFormat.getFullPath());
assertEquals(FileType.JSON, file1StagedFormat.getFileType());
// change the location to the original file location's parent folder
file1StagedFormat.setLocation(inputFile.getParent());
// the upload endpoints should fail given that the location is not correct
expectStatus(Response.Status.BAD_REQUEST, getBuilder(getAPIv2().path("/home/" + HOME_NAME + "/file_preview_unsaved/" + homeFileName)).buildPost(Entity.json(file1StagedFormat)));
expectStatus(Response.Status.BAD_REQUEST, getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_finish/" + homeFileName)).buildPost(Entity.json(file1StagedFormat)));
fileBody.cleanup();
final HomeFileTool tool = l(HomeFileTool.class);
final FilePath filePath = new FilePath(Arrays.asList("@dremio", "filename"));
// this is the root path for a user when staging files
java.nio.file.Path validRootPathForUser = Paths.get(tool.getStagingLocation(filePath, "json").getParent().toString());
// valid path
assertTrue(tool.validStagingLocation(Path.of(validRootPathForUser.resolve("foo").toString())));
assertFalse(tool.validStagingLocation(Path.of(validRootPathForUser.resolve("foo/../../../../").toString())));
assertFalse(tool.validStagingLocation(Path.of("/invalid/path")));
// one level above the valid root, won't include the username and therefore invalid
assertFalse(tool.validStagingLocation(Path.of(validRootPathForUser.getParent().resolve("foo").toString())));
}