Skip to content

HADOOP-18145.Decompress the ZIP file and retain the original file per… #4036

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
steveloughran merged 18 commits into from
Mar 30, 2022
Merged

HADOOP-18145.Decompress the ZIP file and retain the original file per… #4036

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
3b899b1
HADOOP-18145.Decompress the ZIP file and retain the original file per…
Feb 27, 2022
07c0467
Merge branch 'trunk' into hadoop-unzip
smallzhongfeng Feb 27, 2022
5c3e32f
check style
smallzhongfeng Feb 27, 2022
623843a
Update TestFileUtil.java
smallzhongfeng Feb 28, 2022
23a0996
Update TestFileUtil.java
smallzhongfeng Feb 28, 2022
a11d70b
Update the details section of the code
smallzhongfeng Mar 1, 2022
c1dc8c7
Update FileUtil.java
smallzhongfeng Mar 1, 2022
3915b07
Update FileUtil.java
smallzhongfeng Mar 1, 2022
7732357
Update FileUtil.java
smallzhongfeng Mar 1, 2022
2d75af8
Update FileUtil.java
smallzhongfeng Mar 11, 2022
ec98612
Update TestFileUtil.java
smallzhongfeng Mar 11, 2022
b33a9ae
Update TestFileUtil.java
smallzhongfeng Mar 11, 2022
c6c72e6
add try with resources to close the stream
Mar 11, 2022
544f8c9
Update TestFileUtil.java
smallzhongfeng Mar 24, 2022
eb83019
up to date
Mar 24, 2022
3f43d54
change params
Mar 28, 2022
0ecf06e
add test
Mar 29, 2022
90bb4aa
assign variable
Mar 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 43 additions & 9 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,11 @@
import java.nio.charset.CharsetEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.FileSystems;
import java.nio.file.Files;

import java.util.*;
Comment thread
steveloughran marked this conversation as resolved.
Outdated
import java.nio.file.LinkOption;
import java.util.ArrayList;
import java.util.Enumeration;
Expand All @@ -52,13 +55,13 @@
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;

import org.apache.commons.collections.map.CaseInsensitiveMap;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
Expand Down Expand Up @@ -643,12 +646,12 @@ public static long getDU(File dir) {
*/
public static void unZip(InputStream inputStream, File toDir)
throws IOException {
try (ZipInputStream zip = new ZipInputStream(inputStream)) {
try (ZipArchiveInputStream zip = new ZipArchiveInputStream(inputStream)) {
int numOfFailedLastModifiedSet = 0;
String targetDirPath = toDir.getCanonicalPath() + File.separator;
for(ZipEntry entry = zip.getNextEntry();
for(ZipArchiveEntry entry = zip.getNextZipEntry();
entry != null;
entry = zip.getNextEntry()) {
entry = zip.getNextZipEntry()) {
if (!entry.isDirectory()) {
File file = new File(toDir, entry.getName());
if (!file.getCanonicalPath().startsWith(targetDirPath)) {
Expand All @@ -667,6 +670,9 @@ public static void unZip(InputStream inputStream, File toDir)
if (!file.setLastModified(entry.getTime())) {
numOfFailedLastModifiedSet++;
}
if (entry.getPlatform() == ZipArchiveEntry.PLATFORM_UNIX) {
Files.setPosixFilePermissions(file.toPath(), permissionsFromMode(entry.getUnixMode()));
Comment thread
steveloughran marked this conversation as resolved.
}
}
}
if (numOfFailedLastModifiedSet > 0) {
Expand All @@ -676,6 +682,31 @@ public static void unZip(InputStream inputStream, File toDir)
}
}

public static Set<PosixFilePermission> permissionsFromMode(Integer mode) {
Comment thread
steveloughran marked this conversation as resolved.
Outdated
EnumSet<PosixFilePermission> permissions =
EnumSet.noneOf(PosixFilePermission.class);
addPermissions(permissions, "OTHERS", (long) mode);
addPermissions(permissions, "GROUP", (long) mode >> 3);
addPermissions(permissions, "OWNER", (long) mode >> 6);
return permissions;
}

/** Assign the original permissions to the file */
Comment thread
steveloughran marked this conversation as resolved.
Outdated
public static void addPermissions(
Set<PosixFilePermission> permissions,
String prefix,
Long mode) {
Comment thread
steveloughran marked this conversation as resolved.
Outdated
if ((mode & 1L) == 1L) {
permissions.add(PosixFilePermission.valueOf(prefix + "_EXECUTE"));
}
if ((mode & 2L) == 2L) {
permissions.add(PosixFilePermission.valueOf(prefix + "_WRITE"));
}
if ((mode & 4L) == 4L) {
permissions.add(PosixFilePermission.valueOf(prefix + "_READ"));
}
}

/**
* Given a File input it will unzip it in the unzip directory.
* passed as the second parameter
Expand All @@ -684,14 +715,14 @@ public static void unZip(InputStream inputStream, File toDir)
* @throws IOException An I/O exception has occurred
*/
public static void unZip(File inFile, File unzipDir) throws IOException {
Enumeration<? extends ZipEntry> entries;
Enumeration<? extends ZipArchiveEntry> entries;
ZipFile zipFile = new ZipFile(inFile);

try {
entries = zipFile.entries();
entries = zipFile.getEntries();
String targetDirPath = unzipDir.getCanonicalPath() + File.separator;
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
ZipArchiveEntry entry = entries.nextElement();
if (!entry.isDirectory()) {
InputStream in = zipFile.getInputStream(entry);
try {
Expand All @@ -715,6 +746,9 @@ public static void unZip(File inFile, File unzipDir) throws IOException {
}
} finally {
out.close();
if (entry.getPlatform() == ZipArchiveEntry.PLATFORM_UNIX) {
Comment thread
steveloughran marked this conversation as resolved.
Outdated
Files.setPosixFilePermissions(file.toPath(), permissionsFromMode(entry.getUnixMode()));
}
}
} finally {
in.close();
Expand Down
28 changes: 16 additions & 12 deletions hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,11 @@
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.test.GenericTestUtils;
Expand Down Expand Up @@ -706,15 +706,16 @@ public void testCreateLocalTempFile() throws IOException {
public void testUnZip() throws IOException {
// make sa simple zip
final File simpleZip = new File(del, FILE);
OutputStream os = new FileOutputStream(simpleZip);
ZipOutputStream tos = new ZipOutputStream(os);
OutputStream os = new FileOutputStream(simpleZip);
ZipArchiveOutputStream tos = new ZipArchiveOutputStream(os);
Comment thread
smallzhongfeng marked this conversation as resolved.
Outdated
try {
ZipEntry ze = new ZipEntry("foo");
ZipArchiveEntry ze = new ZipArchiveEntry("foo");
ze.setUnixMode(0755);
byte[] data = "some-content".getBytes("UTF-8");
ze.setSize(data.length);
tos.putNextEntry(ze);
tos.putArchiveEntry(ze);
tos.write(data);
tos.closeEntry();
tos.closeArchiveEntry();
tos.flush();
tos.finish();
} finally {
Expand All @@ -726,7 +727,10 @@ public void testUnZip() throws IOException {
// check result:
assertTrue(new File(tmp, "foo").exists());
assertEquals(12, new File(tmp, "foo").length());

assertTrue(new File(tmp, "foo").canExecute());
Comment thread
steveloughran marked this conversation as resolved.
Outdated
assertTrue(new File(tmp, "foo").canRead());
assertTrue(new File(tmp, "foo").canWrite());
Comment thread
steveloughran marked this conversation as resolved.
Outdated

final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog");
regularFile.createNewFile();
assertTrue(regularFile.exists());
Expand All @@ -743,14 +747,14 @@ public void testUnZip2() throws IOException {
// make a simple zip
final File simpleZip = new File(del, FILE);
OutputStream os = new FileOutputStream(simpleZip);
try (ZipOutputStream tos = new ZipOutputStream(os)) {
try (ZipArchiveOutputStream tos = new ZipArchiveOutputStream(os)) {
// Add an entry that contains invalid filename
ZipEntry ze = new ZipEntry("../foo");
ZipArchiveEntry ze = new ZipArchiveEntry("../foo");
byte[] data = "some-content".getBytes(StandardCharsets.UTF_8);
ze.setSize(data.length);
tos.putNextEntry(ze);
tos.putArchiveEntry(ze);
tos.write(data);
tos.closeEntry();
tos.closeArchiveEntry();
tos.flush();
tos.finish();
}
Expand Down