tornadocontrib/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change keystore to version 3

Browse Source 
* Change password protection crypto in keystore to version 3
* Update KeyStoreTests/basic_tests.json
* Add support for PBKDF2 with HMAC-SHA256
* Change MAC and encryption key to avoid unnecessary hashing
* Add tests for test vectors in new wiki page defining version 3
* Add tests for new keystore tests in ethereum/tests repo
* Move JSON loading util to common for use in both tests and
  crypto packages
* Add backwards compatibility with key store version 1
This commit is contained in:
Gustav Simonsson 2015-05-24 03:42:10 +02:00
parent 22c7ce0162
commit d23ec6c419
8 changed files with 385 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
common/test_utils.go Normal file
View File

@ -0,0 +1,37 @@
package common
import (
"encoding/json"
"fmt"
"io/ioutil"
)
// LoadJSON reads the given file and unmarshals its content.
func LoadJSON(file string, val interface{}) error {
content, err := ioutil.ReadFile(file)
if err != nil {
return err
}
if err := json.Unmarshal(content, val); err != nil {
if syntaxerr, ok := err.(*json.SyntaxError); ok {
line := findLine(content, syntaxerr.Offset)
return fmt.Errorf("JSON syntax error at %v:%v: %v", file, line, err)
}
return fmt.Errorf("JSON unmarshal error in %v: %v", file, err)
}
return nil
}
// findLine returns the line number for the given offset into data.
func findLine(data []byte, offset int64) (line int) {
line = 1
for i, r := range string(data) {
if int64(i) >= offset {
return
}
if r == '\n' {
line++
}
}
return
}

26
crypto/crypto.go
View File

@ -258,19 +258,31 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
return key, err
}
func aesCBCDecrypt(key []byte, cipherText []byte, iv []byte) (plainText []byte, err error) {
// AES-128 is selected due to size of encryptKey
func aesCTRXOR(key, inText, iv []byte) ([]byte, error) {
aesBlock, err := aes.NewCipher(key)
if err != nil {
return plainText, err
return nil, err
}
stream := cipher.NewCTR(aesBlock, iv)
outText := make([]byte, len(inText))
stream.XORKeyStream(outText, inText)
return outText, err
}
func aesCBCDecrypt(key, cipherText, iv []byte) ([]byte, error) {
aesBlock, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
decrypter := cipher.NewCBCDecrypter(aesBlock, iv)
paddedPlainText := make([]byte, len(cipherText))
decrypter.CryptBlocks(paddedPlainText, cipherText)
plainText = PKCS7Unpad(paddedPlainText)
if plainText == nil {
paddedPlaintext := make([]byte, len(cipherText))
decrypter.CryptBlocks(paddedPlaintext, cipherText)
plaintext := PKCS7Unpad(paddedPlaintext)
if plaintext == nil {
err = errors.New("Decryption failed: PKCS7Unpad failed after AES decryption")
}
return plainText, err
return plaintext, err
}
// From https://leanpub.com/gocrypto/read#leanpub-auto-block-cipher-modes

26
crypto/key.go
View File

@ -35,7 +35,7 @@ import (
)
const (
version = "1"
version = 3
)
type Key struct {
@ -51,10 +51,17 @@ type plainKeyJSON struct {
Address string `json:"address"`
PrivateKey string `json:"privatekey"`
Id string `json:"id"`
Version string `json:"version"`
Version int `json:"version"`
}
type encryptedKeyJSON struct {
type encryptedKeyJSONV3 struct {
Address string `json:"address"`
Crypto cryptoJSON
Id string `json:"id"`
Version int `json:"version"`
}
type encryptedKeyJSONV1 struct {
Address string `json:"address"`
Crypto cryptoJSON
Id string `json:"id"`
@ -62,13 +69,12 @@ type encryptedKeyJSON struct {
}
type cryptoJSON struct {
Cipher string `json:"cipher"`
CipherText string `json:"ciphertext"`
CipherParams cipherparamsJSON `json:"cipherparams"`
KDF string `json:"kdf"`
KDFParams scryptParamsJSON `json:"kdfparams"`
MAC string `json:"mac"`
Version string `json:"version"`
Cipher string `json:"cipher"`
CipherText string `json:"ciphertext"`
CipherParams cipherparamsJSON `json:"cipherparams"`
KDF string `json:"kdf"`
KDFParams map[string]interface{} `json:"kdfparams"`
MAC string `json:"mac"`
}
type cipherparamsJSON struct {

199
crypto/key_store_passphrase.go
View File

@ -26,40 +26,7 @@
This key store behaves as KeyStorePlain with the difference that
the private key is encrypted and on disk uses another JSON encoding.
Cryptography:
1. Encryption key is first 16 bytes of scrypt derived key
from user passphrase. Scrypt parameters
(work factors) [1][2] are defined as constants below.
2. Scrypt salt is 32 random bytes from CSPRNG.
It's stored in plain next in the key file.
3. MAC is SHA3-256 of concatenation of ciphertext and
last 16 bytes of scrypt derived key.
4. Plaintext is the EC private key bytes.
5. Encryption algo is AES 128 CBC [3][4]
6. CBC IV is 16 random bytes from CSPRNG.
It's stored in plain next in the key file.
7. Plaintext padding is PKCS #7 [5][6]
Encoding:
1. On disk, the ciphertext, MAC, salt and IV are encoded in a JSON object.
cat a key file to see the structure.
2. byte arrays are base64 JSON strings.
3. The EC private key bytes are in uncompressed form [7].
They are a big-endian byte slice of the absolute value of D [8][9].
References:
1. http://www.tarsnap.com/scrypt/scrypt-slides.pdf
2. http://stackoverflow.com/questions/11126315/what-are-optimal-scrypt-work-factors
3. http://en.wikipedia.org/wiki/Advanced_Encryption_Standard
4. http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher-block_chaining_.28CBC.29
5. https://leanpub.com/gocrypto/read#leanpub-auto-block-cipher-modes
6. http://tools.ietf.org/html/rfc2315
7. http://bitcoin.stackexchange.com/questions/3059/what-is-a-compressed-bitcoin-key
8. http://golang.org/pkg/crypto/ecdsa/#PrivateKey
9. https://golang.org/pkg/math/big/#Int.Bytes
The crypto is documented at https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition
*/
@ -68,23 +35,25 @@ package crypto
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"code.google.com/p/go-uuid/uuid"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto/randentropy"
"golang.org/x/crypto/pbkdf2"
"golang.org/x/crypto/scrypt"
)
const (
keyHeaderVersion = "1"
keyHeaderKDF = "scrypt"
keyHeaderKDF = "scrypt"
// 2^18 / 8 / 1 uses 256MB memory and approx 1s CPU time on a modern CPU.
scryptN = 1 << 18
scryptr = 8
@ -105,7 +74,7 @@ func (ks keyStorePassphrase) GenerateNewKey(rand io.Reader, auth string) (key *K
}
func (ks keyStorePassphrase) GetKey(keyAddr common.Address, auth string) (key *Key, err error) {
keyBytes, keyId, err := DecryptKey(ks, keyAddr, auth)
keyBytes, keyId, err := DecryptKeyFromFile(ks, keyAddr, auth)
if err != nil {
return nil, err
}
@ -129,51 +98,43 @@ func (ks keyStorePassphrase) StoreKey(key *Key, auth string) (err error) {
return err
}
encryptKey := Sha3(derivedKey[:16])[:16]
encryptKey := derivedKey[:16]
keyBytes := FromECDSA(key.PrivateKey)
toEncrypt := PKCS7Pad(keyBytes)
AES128Block, err := aes.NewCipher(encryptKey)
iv := randentropy.GetEntropyCSPRNG(aes.BlockSize) // 16
cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
if err != nil {
return err
}
iv := randentropy.GetEntropyCSPRNG(aes.BlockSize) // 16
AES128CBCEncrypter := cipher.NewCBCEncrypter(AES128Block, iv)
cipherText := make([]byte, len(toEncrypt))
AES128CBCEncrypter.CryptBlocks(cipherText, toEncrypt)
mac := Sha3(derivedKey[16:32], cipherText)
scryptParamsJSON := scryptParamsJSON{
N: scryptN,
R: scryptr,
P: scryptp,
DkLen: scryptdkLen,
Salt: hex.EncodeToString(salt),
}
scryptParamsJSON := make(map[string]interface{}, 5)
scryptParamsJSON["n"] = scryptN
scryptParamsJSON["r"] = scryptr
scryptParamsJSON["p"] = scryptp
scryptParamsJSON["dklen"] = scryptdkLen
scryptParamsJSON["salt"] = hex.EncodeToString(salt)
cipherParamsJSON := cipherparamsJSON{
IV: hex.EncodeToString(iv),
}
cryptoStruct := cryptoJSON{
Cipher: "aes-128-cbc",
Cipher: "aes-128-ctr",
CipherText: hex.EncodeToString(cipherText),
CipherParams: cipherParamsJSON,
KDF: "scrypt",
KDFParams: scryptParamsJSON,
MAC: hex.EncodeToString(mac),
Version: "1",
}
encryptedKeyJSON := encryptedKeyJSON{
encryptedKeyJSONV3 := encryptedKeyJSONV3{
hex.EncodeToString(key.Address[:]),
cryptoStruct,
key.Id.String(),
version,
}
keyJSON, err := json.Marshal(encryptedKeyJSON)
keyJSON, err := json.Marshal(encryptedKeyJSONV3)
if err != nil {
return err
}
@ -183,7 +144,7 @@ func (ks keyStorePassphrase) StoreKey(key *Key, auth string) (err error) {
func (ks keyStorePassphrase) DeleteKey(keyAddr common.Address, auth string) (err error) {
// only delete if correct passphrase is given
_, _, err = DecryptKey(ks, keyAddr, auth)
_, _, err = DecryptKeyFromFile(ks, keyAddr, auth)
if err != nil {
return err
}
@ -192,17 +153,43 @@ func (ks keyStorePassphrase) DeleteKey(keyAddr common.Address, auth string) (err
return os.RemoveAll(keyDirPath)
}
func DecryptKey(ks keyStorePassphrase, keyAddr common.Address, auth string) (keyBytes []byte, keyId []byte, err error) {
func DecryptKeyFromFile(ks keyStorePassphrase, keyAddr common.Address, auth string) (keyBytes []byte, keyId []byte, err error) {
fileContent, err := GetKeyFile(ks.keysDirPath, keyAddr)
if err != nil {
return nil, nil, err
}
keyProtected := new(encryptedKeyJSON)
err = json.Unmarshal(fileContent, keyProtected)
m := make(map[string]interface{})
err = json.Unmarshal(fileContent, &m)
v := reflect.ValueOf(m["version"])
if v.Kind() == reflect.String && v.String() == "1" {
k := new(encryptedKeyJSONV1)
err := json.Unmarshal(fileContent, k)
if err != nil {
return nil, nil, err
}
return decryptKeyV1(k, auth)
} else {
k := new(encryptedKeyJSONV3)
err := json.Unmarshal(fileContent, k)
if err != nil {
return nil, nil, err
}
return decryptKeyV3(k, auth)
}
}
func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
if keyProtected.Version != version {
return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
}
if keyProtected.Crypto.Cipher != "aes-128-ctr" {
return nil, nil, fmt.Errorf("Cipher not supported: %v", keyProtected.Crypto.Cipher)
}
keyId = uuid.Parse(keyProtected.Id)
mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
if err != nil {
return nil, nil, err
@ -218,27 +205,49 @@ func DecryptKey(ks keyStorePassphrase, keyAddr common.Address, auth string) (key
return nil, nil, err
}
salt, err := hex.DecodeString(keyProtected.Crypto.KDFParams.Salt)
if err != nil {
return nil, nil, err
}
n := keyProtected.Crypto.KDFParams.N
r := keyProtected.Crypto.KDFParams.R
p := keyProtected.Crypto.KDFParams.P
dkLen := keyProtected.Crypto.KDFParams.DkLen
authArray := []byte(auth)
derivedKey, err := scrypt.Key(authArray, salt, n, r, p, dkLen)
derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
if err != nil {
return nil, nil, err
}
calculatedMAC := Sha3(derivedKey[16:32], cipherText)
if !bytes.Equal(calculatedMAC, mac) {
err = errors.New("Decryption failed: MAC mismatch")
return nil, nil, errors.New("Decryption failed: MAC mismatch")
}
plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
if err != nil {
return nil, nil, err
}
return plainText, keyId, err
}
func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byte, keyId []byte, err error) {
keyId = uuid.Parse(keyProtected.Id)
mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
if err != nil {
return nil, nil, err
}
iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
if err != nil {
return nil, nil, err
}
cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
if err != nil {
return nil, nil, err
}
derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
if err != nil {
return nil, nil, err
}
calculatedMAC := Sha3(derivedKey[16:32], cipherText)
if !bytes.Equal(calculatedMAC, mac) {
return nil, nil, errors.New("Decryption failed: MAC mismatch")
}
plainText, err := aesCBCDecrypt(Sha3(derivedKey[:16])[:16], cipherText, iv)
if err != nil {
@ -246,3 +255,41 @@ func DecryptKey(ks keyStorePassphrase, keyAddr common.Address, auth string) (key
}
return plainText, keyId, err
}
func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) {
authArray := []byte(auth)
salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
if err != nil {
return nil, err
}
dkLen := ensureInt(cryptoJSON.KDFParams["dklen"])
if cryptoJSON.KDF == "scrypt" {
n := ensureInt(cryptoJSON.KDFParams["n"])
r := ensureInt(cryptoJSON.KDFParams["r"])
p := ensureInt(cryptoJSON.KDFParams["p"])
return scrypt.Key(authArray, salt, n, r, p, dkLen)
} else if cryptoJSON.KDF == "pbkdf2" {
c := ensureInt(cryptoJSON.KDFParams["c"])
prf := cryptoJSON.KDFParams["prf"].(string)
if prf != "hmac-sha256" {
return nil, fmt.Errorf("Unsupported PBKDF2 PRF: ", prf)
}
key := pbkdf2.Key(authArray, salt, c, dkLen, sha256.New)
return key, nil
}
return nil, fmt.Errorf("Unsupported KDF: ", cryptoJSON.KDF)
}
// TODO: can we do without this when unmarshalling dynamic JSON?
// why do integers in KDF params end up as float64 and not int after
// unmarshal?
func ensureInt(x interface{}) int {
res, ok := x.(int)
if !ok {
res = int(x.(float64))
}
return res
}

114
crypto/key_store_test.go
View File

@ -1,10 +1,13 @@
package crypto
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto/randentropy"
"encoding/hex"
"fmt"
"reflect"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto/randentropy"
)
func TestKeyStorePlain(t *testing.T) {
@ -97,3 +100,110 @@ func TestImportPreSaleKey(t *testing.T) {
t.Fatal(err)
}
}
// Test and utils for the key store tests in the Ethereum JSON tests;
// tests/KeyStoreTests/basic_tests.json
type KeyStoreTestV3 struct {
Json encryptedKeyJSONV3
Password string
Priv string
}
type KeyStoreTestV1 struct {
Json encryptedKeyJSONV1
Password string
Priv string
}
func TestV3_PBKDF2_1(t *testing.T) {
tests := loadKeyStoreTestV3("tests/v3_test_vector.json", t)
testDecryptV3(tests["wikipage_test_vector_pbkdf2"], t)
}
func TestV3_PBKDF2_2(t *testing.T) {
tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
testDecryptV3(tests["test1"], t)
}
func TestV3_PBKDF2_3(t *testing.T) {
tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
testDecryptV3(tests["python_generated_test_with_odd_iv"], t)
}
func TestV3_PBKDF2_4(t *testing.T) {
tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
testDecryptV3(tests["evilnonce"], t)
}
func TestV3_Scrypt_1(t *testing.T) {
tests := loadKeyStoreTestV3("tests/v3_test_vector.json", t)
testDecryptV3(tests["wikipage_test_vector_scrypt"], t)
}
func TestV3_Scrypt_2(t *testing.T) {
tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
testDecryptV3(tests["test2"], t)
}
func TestV1_1(t *testing.T) {
tests := loadKeyStoreTestV1("tests/v1_test_vector.json", t)
testDecryptV1(tests["test1"], t)
}
func TestV1_2(t *testing.T) {
ks := NewKeyStorePassphrase("tests/v1")
addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e")
k, err := ks.GetKey(addr, "g")
if err != nil {
t.Fatal(err)
}
if k.Address != addr {
t.Fatal(fmt.Errorf("Unexpected address: %v, expected %v", k.Address, addr))
}
privHex := hex.EncodeToString(FromECDSA(k.PrivateKey))
expectedHex := "d1b1178d3529626a1a93e073f65028370d14c7eb0936eb42abef05db6f37ad7d"
if privHex != expectedHex {
t.Fatal(fmt.Errorf("Unexpected privkey: %v, expected %v", privHex, expectedHex))
}
}
func testDecryptV3(test KeyStoreTestV3, t *testing.T) {
privBytes, _, err := decryptKeyV3(&test.Json, test.Password)
if err != nil {
t.Fatal(err)
}
privHex := hex.EncodeToString(privBytes)
if test.Priv != privHex {
t.Fatal(fmt.Errorf("Decrypted bytes not equal to test, expected %v have %v", test.Priv, privHex))
}
}
func testDecryptV1(test KeyStoreTestV1, t *testing.T) {
privBytes, _, err := decryptKeyV1(&test.Json, test.Password)
if err != nil {
t.Fatal(err)
}
privHex := hex.EncodeToString(privBytes)
if test.Priv != privHex {
t.Fatal(fmt.Errorf("Decrypted bytes not equal to test, expected %v have %v", test.Priv, privHex))
}
}
func loadKeyStoreTestV3(file string, t *testing.T) map[string]KeyStoreTestV3 {
tests := make(map[string]KeyStoreTestV3)
err := common.LoadJSON(file, &tests)
if err != nil {
t.Fatal(err)
}
return tests
}
func loadKeyStoreTestV1(file string, t *testing.T) map[string]KeyStoreTestV1 {
tests := make(map[string]KeyStoreTestV1)
err := common.LoadJSON(file, &tests)
if err != nil {
t.Fatal(err)
}
return tests
}

1
crypto/tests/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e Normal file
View File

@ -0,0 +1 @@
{"address":"cb61d5a9c4896fb9658090b597ef0e7be6f7b67e","Crypto":{"cipher":"aes-128-cbc","ciphertext":"6143d3192db8b66eabd693d9c4e414dcfaee52abda451af79ccf474dafb35f1bfc7ea013aa9d2ee35969a1a2e8d752d0","cipherparams":{"iv":"35337770fc2117994ecdcad026bccff4"},"kdf":"scrypt","kdfparams":{"n":262144,"r":8,"p":1,"dklen":32,"salt":"9afcddebca541253a2f4053391c673ff9fe23097cd8555d149d929e4ccf1257f"},"mac":"3f3d5af884b17a100b0b3232c0636c230a54dc2ac8d986227219b0dd89197644","version":"1"},"id":"e25f7c1f-d318-4f29-b62c-687190d4d299","version":"1"}

28
crypto/tests/v1_test_vector.json Normal file
View File

@ -0,0 +1,28 @@
{
"test1": {
"json": {
"Crypto": {
"cipher": "aes-128-cbc",
"cipherparams": {
"iv": "35337770fc2117994ecdcad026bccff4"
},
"ciphertext": "6143d3192db8b66eabd693d9c4e414dcfaee52abda451af79ccf474dafb35f1bfc7ea013aa9d2ee35969a1a2e8d752d0",
"kdf": "scrypt",
"kdfparams": {
"dklen": 32,
"n": 262144,
"p": 1,
"r": 8,
"salt": "9afcddebca541253a2f4053391c673ff9fe23097cd8555d149d929e4ccf1257f"
},
"mac": "3f3d5af884b17a100b0b3232c0636c230a54dc2ac8d986227219b0dd89197644",
"version": "1"
},
"address": "cb61d5a9c4896fb9658090b597ef0e7be6f7b67e",
"id": "e25f7c1f-d318-4f29-b62c-687190d4d299",
"version": "1"
},
"password": "g",
"priv": "d1b1178d3529626a1a93e073f65028370d14c7eb0936eb42abef05db6f37ad7d"
}
}

49
crypto/tests/v3_test_vector.json Normal file
View File

@ -0,0 +1,49 @@
{
"wikipage_test_vector_scrypt": {
"json": {
"crypto" : {
"cipher" : "aes-128-ctr",
"cipherparams" : {
"iv" : "83dbcc02d8ccb40e466191a123791e0e"
},
"ciphertext" : "d172bf743a674da9cdad04534d56926ef8358534d458fffccd4e6ad2fbde479c",
"kdf" : "scrypt",
"kdfparams" : {
"dklen" : 32,
"n" : 262144,
"r" : 1,
"p" : 8,
"salt" : "ab0c7876052600dd703518d6fc3fe8984592145b591fc8fb5c6d43190334ba19"
},
"mac" : "2103ac29920d71da29f15d75b4a16dbe95cfd7ff8faea1056c33131d846e3097"
},
"id" : "3198bc9c-6672-5ab3-d995-4942343ae5b6",
"version" : 3
},
"password": "testpassword",
"priv": "7a28b5ba57c53603b0b07b56bba752f7784bf506fa95edc395f5cf6c7514fe9d"
},
"wikipage_test_vector_pbkdf2": {
"json": {
"crypto" : {
"cipher" : "aes-128-ctr",
"cipherparams" : {
"iv" : "6087dab2f9fdbbfaddc31a909735c1e6"
},
"ciphertext" : "5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46",
"kdf" : "pbkdf2",
"kdfparams" : {
"c" : 262144,
"dklen" : 32,
"prf" : "hmac-sha256",
"salt" : "ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd"
},
"mac" : "517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2"
},
"id" : "3198bc9c-6672-5ab3-d995-4942343ae5b6",
"version" : 3
},
"password": "testpassword",
"priv": "7a28b5ba57c53603b0b07b56bba752f7784bf506fa95edc395f5cf6c7514fe9d"
}
}