By encrypting (and decrypting) all data on the client side we are able to safely hide the content of our repository from the owners/operators the Amazon S3 service, making it a secure backup solution for Git repositories. Signed-off-by: Shawn O. Pearce <spearce@xxxxxxxxxxx> --- .../src/org/spearce/jgit/transport/AmazonS3.java | 82 ++++++++- .../org/spearce/jgit/transport/WalkEncryption.java | 188 ++++++++++++++++++++ 2 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 org.spearce.jgit/src/org/spearce/jgit/transport/WalkEncryption.java diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/AmazonS3.java b/org.spearce.jgit/src/org/spearce/jgit/transport/AmazonS3.java index 466d9e9..4c82967 100644 --- a/org.spearce.jgit/src/org/spearce/jgit/transport/AmazonS3.java +++ b/org.spearce.jgit/src/org/spearce/jgit/transport/AmazonS3.java @@ -37,6 +37,7 @@ package org.spearce.jgit.transport; +import java.io.ByteArrayOutputStream; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; @@ -53,6 +54,7 @@ import java.security.DigestOutputStream; import java.security.InvalidKeyException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; @@ -93,6 +95,11 @@ import org.xml.sax.helpers.XMLReaderFactory; * <p> * Authentication is always performed using the user's AWSAccessKeyId and their * private AWSSecretAccessKey. + * <p> + * Optional client-side encryption may be enabled if requested. The format is + * compatible with <a href="http://jets3t.s3.amazonaws.com/index.html";>jets3t</a>, + * a popular Java based Amazon S3 client library. Enabling encryption can hide + * sensitive data from the operators of the S3 service. */ public class AmazonS3 { private static final Set<String> SIGNED_HEADERS; @@ -103,6 +110,8 @@ public class AmazonS3 { private static final String X_AMZ_ACL = "x-amz-acl"; + private static final String X_AMZ_META = "x-amz-meta-"; + static { SIGNED_HEADERS = new HashSet<String>(); SIGNED_HEADERS.add("content-type"); @@ -161,6 +170,9 @@ public class AmazonS3 { /** Maximum number of times to try an operation. */ private final int maxAttempts; + /** Encryption algorithm, may be a null instance that provides pass-through. */ + private final WalkEncryption encryption; + /** * Create a new S3 client for the supplied user information. * <p> @@ -179,6 +191,10 @@ public class AmazonS3 { * * # Number of times to retry after internal error from S3. * httpclient.retry-max: 3 + * + * # End-to-end encryption (hides content from S3 owners) + * password: <encryption pass-phrase> + * crypto.algorithm: PBEWithMD5AndDES * </pre> * * @param props @@ -207,6 +223,22 @@ public class AmazonS3 { else throw new IllegalArgumentException("Invalid acl: " + pacl); + try { + final String cPas = props.getProperty("password"); + if (cPas != null) { + String cAlg = props.getProperty("crypto.algorithm"); + if (cAlg == null) + cAlg = "PBEWithMD5AndDES"; + encryption = new WalkEncryption.ObjectEncryptionV2(cAlg, cPas); + } else { + encryption = WalkEncryption.NONE; + } + } catch (InvalidKeySpecException e) { + throw new IllegalArgumentException("Invalid encryption", e); + } catch (NoSuchAlgorithmException e) { + throw new IllegalArgumentException("Invalid encryption", e); + } + maxAttempts = Integer.parseInt(props.getProperty( "httpclient.retry-max", "3")); proxySelector = ProxySelector.getDefault(); @@ -232,6 +264,7 @@ public class AmazonS3 { authorize(c); switch (HttpSupport.response(c)) { case HttpURLConnection.HTTP_OK: + encryption.validate(c, X_AMZ_META); return c; case HttpURLConnection.HTTP_NOT_FOUND: throw new FileNotFoundException(key); @@ -245,6 +278,19 @@ public class AmazonS3 { } /** + * Decrypt an input stream from {@link #get(String, String)}. + * + * @param u + * connection previously created by {@link #get(String, String)}}. + * @return stream to read plain text from. + * @throws IOException + * decryption could not be configured. + */ + public InputStream decrypt(final URLConnection u) throws IOException { + return encryption.decrypt(u.getInputStream()); + } + + /** * List the names of keys available within a bucket. * <p> * This method is primarily meant for obtaining a "recursive directory @@ -326,6 +372,16 @@ public class AmazonS3 { */ public void put(final String bucket, final String key, final byte[] data) throws IOException { + if (encryption != WalkEncryption.NONE) { + // We have to copy to produce the cipher text anyway so use + // the large object code path as it supports that behavior. + // + final OutputStream os = beginPut(bucket, key); + os.write(data); + os.close(); + return; + } + final String md5str = Base64.encodeBytes(newMD5().digest(data)); final String lenstr = String.valueOf(data.length); for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) { @@ -375,8 +431,11 @@ public class AmazonS3 { * @param key * key of the object within its bucket. * @return a stream which accepts the new data, and transmits once closed. + * @throws IOException + * if encryption was enabled it could not be configured. */ - public OutputStream beginPut(final String bucket, final String key) { + public OutputStream beginPut(final String bucket, final String key) + throws IOException { final MessageDigest md5 = newMD5(); final TemporaryBuffer buffer = new TemporaryBuffer() { @Override @@ -389,7 +448,7 @@ public class AmazonS3 { } } }; - return new DigestOutputStream(buffer, md5); + return encryption.encrypt(new DigestOutputStream(buffer, md5)); } private void putImpl(final String bucket, final String key, @@ -402,6 +461,7 @@ public class AmazonS3 { c.setRequestProperty("Content-Length", lenstr); c.setRequestProperty("Content-MD5", md5str); c.setRequestProperty(X_AMZ_ACL, acl); + encryption.request(c, X_AMZ_META); authorize(c); c.setDoOutput(true); c.setFixedLengthStreamingMode((int) len); @@ -426,8 +486,22 @@ public class AmazonS3 { private IOException error(final String action, final String key, final HttpURLConnection c) throws IOException { - return new IOException(action + " of '" + key + "' failed: " - + HttpSupport.response(c) + " " + c.getResponseMessage()); + final IOException err = new IOException(action + " of '" + key + + "' failed: " + HttpSupport.response(c) + " " + + c.getResponseMessage()); + final ByteArrayOutputStream b = new ByteArrayOutputStream(); + byte[] buf = new byte[2048]; + for (;;) { + final int n = c.getErrorStream().read(buf); + if (n < 0) + break; + if (n > 0) + b.write(buf, 0, n); + } + buf = b.toByteArray(); + if (buf.length > 0) + err.initCause(new IOException("\n" + new String(buf))); + return err; } private IOException maxAttempts(final String action, final String key) { diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/WalkEncryption.java b/org.spearce.jgit/src/org/spearce/jgit/transport/WalkEncryption.java new file mode 100644 index 0000000..cec6d75 --- /dev/null +++ b/org.spearce.jgit/src/org/spearce/jgit/transport/WalkEncryption.java @@ -0,0 +1,188 @@ +/* + * Copyright (C) 2008, Shawn O. Pearce <spearce@xxxxxxxxxxx> + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Git Development Community nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.spearce.jgit.transport; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; + +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.CipherOutputStream; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.PBEParameterSpec; + +abstract class WalkEncryption { + static final WalkEncryption NONE = new NoEncryption(); + + static final String JETS3T_CRYPTO_VER = "jets3t-crypto-ver"; + + static final String JETS3T_CRYPTO_ALG = "jets3t-crypto-alg"; + + abstract OutputStream encrypt(OutputStream os) throws IOException; + + abstract InputStream decrypt(InputStream in) throws IOException; + + abstract void request(HttpURLConnection u, String prefix); + + abstract void validate(HttpURLConnection u, String p) throws IOException; + + protected void validateImpl(final HttpURLConnection u, final String p, + final String version, final String name) throws IOException { + String v; + + v = u.getHeaderField(p + JETS3T_CRYPTO_VER); + if (v == null) + v = ""; + if (!version.equals(v)) + throw new IOException("Unsupported encryption version: " + v); + + v = u.getHeaderField(p + JETS3T_CRYPTO_ALG); + if (v == null) + v = ""; + if (!name.equals(v)) + throw new IOException("Unsupported encryption algorithm: " + v); + } + + IOException error(final Throwable why) { + final IOException e; + e = new IOException("Encryption error: " + why.getMessage()); + e.initCause(why); + return e; + } + + private static class NoEncryption extends WalkEncryption { + @Override + void request(HttpURLConnection u, String prefix) { + // Don't store any request properties. + } + + @Override + void validate(final HttpURLConnection u, final String p) + throws IOException { + validateImpl(u, p, "", ""); + } + + @Override + InputStream decrypt(InputStream in) { + return in; + } + + @Override + OutputStream encrypt(OutputStream os) { + return os; + } + } + + static class ObjectEncryptionV2 extends WalkEncryption { + private static int ITERATION_COUNT = 5000; + + private static byte[] salt = { (byte) 0xA4, (byte) 0x0B, (byte) 0xC8, + (byte) 0x34, (byte) 0xD6, (byte) 0x95, (byte) 0xF3, (byte) 0x13 }; + + private final String algorithmName; + + private final SecretKey skey; + + private final PBEParameterSpec aspec; + + ObjectEncryptionV2(final String algo, final String key) + throws InvalidKeySpecException, NoSuchAlgorithmException { + algorithmName = algo; + + final PBEKeySpec s; + s = new PBEKeySpec(key.toCharArray(), salt, ITERATION_COUNT, 32); + skey = SecretKeyFactory.getInstance(algo).generateSecret(s); + aspec = new PBEParameterSpec(salt, ITERATION_COUNT); + } + + @Override + void request(final HttpURLConnection u, final String prefix) { + u.setRequestProperty(prefix + JETS3T_CRYPTO_VER, "2"); + u.setRequestProperty(prefix + JETS3T_CRYPTO_ALG, algorithmName); + } + + @Override + void validate(final HttpURLConnection u, final String p) + throws IOException { + validateImpl(u, p, "2", algorithmName); + } + + @Override + OutputStream encrypt(final OutputStream os) throws IOException { + try { + final Cipher c = Cipher.getInstance(algorithmName); + c.init(Cipher.ENCRYPT_MODE, skey, aspec); + return new CipherOutputStream(os, c); + } catch (NoSuchAlgorithmException e) { + throw error(e); + } catch (NoSuchPaddingException e) { + throw error(e); + } catch (InvalidKeyException e) { + throw error(e); + } catch (InvalidAlgorithmParameterException e) { + throw error(e); + } + } + + @Override + InputStream decrypt(final InputStream in) throws IOException { + try { + final Cipher c = Cipher.getInstance(algorithmName); + c.init(Cipher.DECRYPT_MODE, skey, aspec); + return new CipherInputStream(in, c); + } catch (NoSuchAlgorithmException e) { + throw error(e); + } catch (NoSuchPaddingException e) { + throw error(e); + } catch (InvalidKeyException e) { + throw error(e); + } catch (InvalidAlgorithmParameterException e) { + throw error(e); + } + } + } +} -- 1.5.6.74.g8a5e -- To unsubscribe from this list: send the line "unsubscribe git" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html
