[Gluster-devel] [PATCH BUG:393 01/10] transport/encryption: add infrastructure
Corentin Chary
corentin.chary at gmail.com
Wed Nov 18 11:13:53 UTC 2009
Add support for transport/encryption:
* Modify gf_common_hdr to add a crypto field
* Modify current transports to keep trace of the encryption xl
* Add a buflen parameter to submit/receive hook, because the
encrytion xlator need to know the buffer size
Signed-off-by: Corentin Chary <corentin.chary at gmail.com>
---
libglusterfs/src/Makefile.am | 2 +-
libglusterfs/src/protocol.h | 3 +-
libglusterfs/src/transport.c | 253 +++++++++++++++++++------
libglusterfs/src/transport.h | 25 +++-
transport/Makefile.am | 4 +-
transport/ib-verbs/src/ib-verbs.c | 3 +
transport/socket/src/socket.c | 6 +-
xlators/protocol/client/src/client-protocol.c | 3 +
xlators/protocol/server/src/server-protocol.c | 3 +
9 files changed, 233 insertions(+), 69 deletions(-)
diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index 90f8b65..bca26e8 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -1,6 +1,6 @@
libglusterfs_la_CFLAGS = -fPIC -Wall -g -shared -nostartfiles $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS)
-libglusterfs_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D_GNU_SOURCE -DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" -DSCHEDULERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler\" -DTRANSPORTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/transport\" -D$(GF_HOST_OS) -DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" -I$(CONTRIBDIR)/rbtree
+libglusterfs_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D_GNU_SOURCE -DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" -DSCHEDULERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler\" -DTRANSPORTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/transport\" -DTRANSPORTCRYPTODIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/transport/encryption\" -D$(GF_HOST_OS) -DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" -I$(top_srcdir)/contrib/rbtree
libglusterfs_la_LIBADD = @LEXLIB@
diff --git a/libglusterfs/src/protocol.h b/libglusterfs/src/protocol.h
index 8faf5f3..2341ec8 100644
--- a/libglusterfs/src/protocol.h
+++ b/libglusterfs/src/protocol.h
@@ -963,12 +963,12 @@ typedef struct {
uint32_t op_errno;
} __attribute__ ((packed)) gf_hdr_rsp_t;
-
typedef struct {
uint64_t callid;
uint32_t type;
uint32_t op;
uint32_t size;
+ uint32_t crypto;
union {
gf_hdr_req_t req;
gf_hdr_rsp_t rsp;
@@ -997,6 +997,7 @@ __gf_hdr_new (int size)
#define gf_hdr_len(type, x) (sizeof (gf_hdr_common_t) + sizeof (*type) + x)
#define gf_hdr_new(type, x) __gf_hdr_new (sizeof (*type) + x)
+#define GF_HDR_COMMON_SIZE sizeof (gf_hdr_common_t)
static inline void *
gf_param (gf_hdr_common_t *hdr)
diff --git a/libglusterfs/src/transport.c b/libglusterfs/src/transport.c
index 5ffcfa4..89c71c1 100644
--- a/libglusterfs/src/transport.c
+++ b/libglusterfs/src/transport.c
@@ -34,30 +34,15 @@
#include "glusterfs.h"
#include "xlator.h"
#include "list.h"
+#include "protocol.h"
-
-transport_t *
-transport_load (dict_t *options,
- xlator_t *xl)
+static void
+backward_compatibility (dict_t *options, xlator_t *xl)
{
- struct transport *trans = NULL, *return_trans = NULL;
char *addr_family = NULL;
- char *name = NULL;
- void *handle = NULL;
char *type = NULL;
- char str[] = "ERROR";
int32_t ret = -1;
int8_t is_tcp = 0, is_unix = 0, is_ibsdp = 0;
- volume_opt_list_t *vol_opt = NULL;
-
- GF_VALIDATE_OR_GOTO("transport", options, fail);
- GF_VALIDATE_OR_GOTO("transport", xl, fail);
-
- trans = CALLOC (1, sizeof (struct transport));
- GF_VALIDATE_OR_GOTO("transport", trans, fail);
-
- trans->xl = xl;
- type = str;
/* Backward compatibility */
ret = dict_get_str (options, "transport-type", &type);
@@ -89,13 +74,13 @@ transport_load (dict_t *options,
} else {
{
/* Backword compatibility to handle * /client,
- * * /server.
+ * * /server.
*/
char *tmp = strchr (type, '/');
if (tmp)
*tmp = '\0';
}
-
+
is_tcp = strcmp (type, "tcp");
is_unix = strcmp (type, "unix");
is_ibsdp = strcmp (type, "ib-sdp");
@@ -103,15 +88,15 @@ transport_load (dict_t *options,
(is_unix == 0) ||
(is_ibsdp == 0)) {
if (is_tcp == 0)
- ret = dict_set_str (options,
+ ret = dict_set_str (options,
"transport.address-family",
"inet");
if (is_unix == 0)
- ret = dict_set_str (options,
+ ret = dict_set_str (options,
"transport.address-family",
"unix");
if (is_ibsdp == 0)
- ret = dict_set_str (options,
+ ret = dict_set_str (options,
"transport.address-family",
"inet-sdp");
@@ -119,27 +104,36 @@ transport_load (dict_t *options,
gf_log ("dict", GF_LOG_DEBUG,
"setting address-family failed");
- ret = dict_set_str (options,
+ ret = dict_set_str (options,
"transport-type", "socket");
if (ret < 0)
gf_log ("dict", GF_LOG_DEBUG,
"setting transport-type failed");
}
}
+}
+
+static int
+transport_init (dict_t *options, xlator_t *xl, struct transport *trans)
+{
+ int ret;
+ void *handle = NULL;
+ char *name = NULL;
+ char *type = "ERROR";
+ volume_opt_list_t *vol_opt = NULL;
ret = dict_get_str (options, "transport-type", &type);
if (ret < 0) {
- FREE (trans);
gf_log ("transport", GF_LOG_ERROR,
"'option transport-type <xx>' missing in volume '%s'",
xl->name);
- goto fail;
+ return ret;
}
ret = asprintf (&name, "%s/%s.so", TRANSPORTDIR, type);
if (-1 == ret) {
gf_log ("transport", GF_LOG_ERROR, "asprintf failed");
- goto fail;
+ return -1;
}
gf_log ("transport", GF_LOG_DEBUG,
"attempt to load file %s", name);
@@ -149,38 +143,34 @@ transport_load (dict_t *options,
gf_log ("transport", GF_LOG_ERROR, "%s", dlerror ());
gf_log ("transport", GF_LOG_ERROR,
"volume '%s': transport-type '%s' is not valid or "
- "not found on this machine",
+ "not found on this machine",
xl->name, type);
FREE (name);
- FREE (trans);
- goto fail;
+ return -1;
}
FREE (name);
-
+
trans->ops = dlsym (handle, "tops");
if (trans->ops == NULL) {
gf_log ("transport", GF_LOG_ERROR,
"dlsym (transport_ops) on %s", dlerror ());
- FREE (trans);
- goto fail;
+ return -1;
}
trans->init = dlsym (handle, "init");
if (trans->init == NULL) {
gf_log ("transport", GF_LOG_ERROR,
"dlsym (gf_transport_init) on %s", dlerror ());
- FREE (trans);
- goto fail;
+ return -1;
}
trans->fini = dlsym (handle, "fini");
if (trans->fini == NULL) {
gf_log ("transport", GF_LOG_ERROR,
"dlsym (gf_transport_fini) on %s", dlerror ());
- FREE (trans);
- goto fail;
+ return -1;
}
-
+
vol_opt = CALLOC (1, sizeof (volume_opt_list_t));
vol_opt->given_opt = dlsym (handle, "options");
if (vol_opt->given_opt == NULL) {
@@ -188,32 +178,140 @@ transport_load (dict_t *options,
"volume option validation not specified");
} else {
list_add_tail (&vol_opt->list, &xl->volume_options);
- if (-1 ==
- validate_xlator_volume_options (xl,
+ if (-1 ==
+ validate_xlator_volume_options (xl,
vol_opt->given_opt)) {
gf_log ("transport", GF_LOG_ERROR,
"volume option validation failed");
- FREE (trans);
- goto fail;
+ return -1;
}
}
-
+
ret = trans->init (trans);
if (ret != 0) {
gf_log ("transport", GF_LOG_ERROR,
"'%s' initialization failed", type);
- FREE (trans);
- goto fail;
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+crypto_init (dict_t *options, xlator_t *xl, struct transport *trans)
+{
+ int ret;
+ void *handle = NULL;
+ char *name = NULL;
+ char *type = "ERROR";
+ volume_opt_list_t *vol_opt = NULL;
+
+ ret = dict_get_str (options, "transport.crypto.type", &type);
+ if (ret < 0)
+ return 0;
+
+ ret = asprintf (&name, "%s/%s.so", TRANSPORTCRYPTODIR, type);
+ if (-1 == ret) {
+ gf_log ("transport", GF_LOG_ERROR, "asprintf failed");
+ return -1;
+ }
+ gf_log ("transport", GF_LOG_DEBUG,
+ "attempt to load file %s", name);
+
+ handle = dlopen (name, RTLD_NOW|RTLD_GLOBAL);
+ if (handle == NULL) {
+ gf_log ("transport", GF_LOG_ERROR, "%s", dlerror ());
+ gf_log ("transport", GF_LOG_ERROR,
+ "volume '%s': transport.crypto.type '%s' is not valid or "
+ "not found on this machine",
+ xl->name, type);
+ FREE (name);
+ return -1;
+ }
+ FREE (name);
+
+ trans->crypto.ops = dlsym (handle, "tcops");
+ if (trans->crypto.ops == NULL) {
+ gf_log ("transport", GF_LOG_ERROR,
+ "dlsym (transport_crypto_ops) on %s", dlerror ());
+ return -1;
+ }
+
+ trans->crypto.init = dlsym (handle, "init");
+ if (trans->crypto.init == NULL) {
+ gf_log ("transport", GF_LOG_ERROR,
+ "dlsym (gf_transport_crypto_init) on %s", dlerror ());
+ return -1;
+ }
+
+ trans->crypto.fini = dlsym (handle, "fini");
+ if (trans->crypto.fini == NULL) {
+ gf_log ("transport", GF_LOG_ERROR,
+ "dlsym (gf_transport_crypto_fini) on %s", dlerror ());
+ return -1;
}
+ vol_opt = CALLOC (1, sizeof (volume_opt_list_t));
+ vol_opt->given_opt = dlsym (handle, "options");
+ if (vol_opt->given_opt == NULL) {
+ gf_log ("transport", GF_LOG_DEBUG,
+ "volume option validation not specified");
+ } else {
+ list_add_tail (&vol_opt->list, &xl->volume_options);
+ if (-1 ==
+ validate_xlator_volume_options (xl,
+ vol_opt->given_opt)) {
+ gf_log ("transport", GF_LOG_ERROR,
+ "volume option validation failed");
+ return -1;
+ }
+ }
+
+ ret = trans->crypto.init (&trans->crypto);
+ if (ret != 0) {
+ gf_log ("transport", GF_LOG_ERROR,
+ "'%s' initialization failed", type);
+ return -1;
+ }
+
+ return 0;
+}
+
+transport_t *
+transport_load (dict_t *options,
+ xlator_t *xl)
+{
+ struct transport *trans = NULL, *return_trans = NULL;
+ int ret;
+
+ GF_VALIDATE_OR_GOTO("transport", options, fail);
+ GF_VALIDATE_OR_GOTO("transport", xl, fail);
+
+ trans = CALLOC (1, sizeof (struct transport));
+ GF_VALIDATE_OR_GOTO("transport", trans, fail);
+
+ trans->xl = xl;
+
+ backward_compatibility (options, xl);
+
+ ret = transport_init (options, xl, trans);
+ if (ret < 0)
+ goto fail_free;
+ ret = crypto_init (options, xl, trans);
+ if (ret < 0)
+ goto fail_free;
+
pthread_mutex_init (&trans->lock, NULL);
return_trans = trans;
fail:
return return_trans;
+fail_free:
+ FREE (trans);
+ return return_trans;
}
-int32_t
+int32_t
transport_submit (transport_t *this, char *buf, int32_t len,
struct iovec *vector, int count,
struct iobref *iobref)
@@ -222,6 +320,16 @@ transport_submit (transport_t *this, char *buf, int32_t len,
transport_t *peer_trans = NULL;
struct iobuf *iobuf = NULL;
struct transport_msg *msg = NULL;
+ gf_hdr_common_t *hdr;
+
+ hdr = (gf_hdr_common_t *) buf;
+ hdr->crypto = hton32 (this->crypto.magic);
+
+ if (this->crypto.ops && this->crypto.ops->encrypt) {
+ ret = this->crypto.ops->encrypt (this, buf, len, vector, count);
+ if (ret)
+ goto fail;
+ }
if (this->peer_trans) {
peer_trans = this->peer_trans;
@@ -244,6 +352,7 @@ transport_submit (transport_t *this, char *buf, int32_t len,
iov_unload (iobuf->ptr, vector, count);
msg->iobuf = iobuf;
+ msg->buflen = iov_length (vector, count);
}
pthread_mutex_lock (&peer_trans->handover.mutex);
@@ -258,20 +367,20 @@ transport_submit (transport_t *this, char *buf, int32_t len,
GF_VALIDATE_OR_GOTO("transport", this, fail);
GF_VALIDATE_OR_GOTO("transport", this->ops, fail);
-
+
ret = this->ops->submit (this, buf, len, vector, count, iobref);
fail:
return ret;
}
-int32_t
+int32_t
transport_connect (transport_t *this)
{
int ret = -1;
-
+
GF_VALIDATE_OR_GOTO("transport", this, fail);
-
+
ret = this->ops->connect (this);
fail:
return ret;
@@ -282,38 +391,41 @@ int32_t
transport_listen (transport_t *this)
{
int ret = -1;
-
+
GF_VALIDATE_OR_GOTO("transport", this, fail);
-
+
ret = this->ops->listen (this);
fail:
return ret;
}
-int32_t
+int32_t
transport_disconnect (transport_t *this)
{
int32_t ret = -1;
-
+
GF_VALIDATE_OR_GOTO("transport", this, fail);
-
+
ret = this->ops->disconnect (this);
fail:
return ret;
}
-int32_t
+int32_t
transport_destroy (transport_t *this)
{
int32_t ret = -1;
GF_VALIDATE_OR_GOTO("transport", this, fail);
-
+
if (this->fini)
this->fini (this);
+ if (this->crypto.fini)
+ this->crypto.fini (&this->crypto);
+
pthread_mutex_destroy (&this->lock);
FREE (this);
fail:
@@ -327,13 +439,13 @@ transport_ref (transport_t *this)
transport_t *return_this = NULL;
GF_VALIDATE_OR_GOTO("transport", this, fail);
-
+
pthread_mutex_lock (&this->lock);
{
this->refcount ++;
}
pthread_mutex_unlock (&this->lock);
-
+
return_this = this;
fail:
return return_this;
@@ -345,6 +457,8 @@ transport_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
struct iobuf **iobuf_p)
{
int32_t ret = -1;
+ size_t buflen;
+ gf_hdr_common_t *hdr;
GF_VALIDATE_OR_GOTO("transport", this, fail);
@@ -352,11 +466,26 @@ transport_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
*hdr_p = this->handover.msg->hdr;
*hdrlen_p = this->handover.msg->hdrlen;
*iobuf_p = this->handover.msg->iobuf;
+ buflen = this->handover.msg->buflen;
return 0;
}
- ret = this->ops->receive (this, hdr_p, hdrlen_p, iobuf_p);
+ ret = this->ops->receive (this, hdr_p, hdrlen_p, iobuf_p, &buflen);
+ if (ret)
+ goto fail;
+
+ hdr = (gf_hdr_common_t *) *hdr_p;
+ hdr->crypto = ntoh32 (hdr->crypto);
+
+ if (hdr->crypto != this->crypto.magic) {
+ ret = -1;
+ goto fail;
+ }
+
+ if (this->crypto.ops && this->crypto.ops->decrypt)
+ ret = this->crypto.ops->decrypt (this, *hdr_p, *hdrlen_p,
+ *iobuf_p, buflen);
fail:
return ret;
}
@@ -369,7 +498,7 @@ transport_unref (transport_t *this)
int32_t ret = -1;
GF_VALIDATE_OR_GOTO("transport", this, fail);
-
+
pthread_mutex_lock (&this->lock);
{
refcount = --this->refcount;
@@ -380,7 +509,7 @@ transport_unref (transport_t *this)
this->xl->notify (this->xl, GF_EVENT_TRANSPORT_CLEANUP, this);
transport_destroy (this);
}
-
+
ret = 0;
fail:
return ret;
diff --git a/libglusterfs/src/transport.h b/libglusterfs/src/transport.h
index f0623d5..393aa7e 100644
--- a/libglusterfs/src/transport.h
+++ b/libglusterfs/src/transport.h
@@ -28,7 +28,10 @@
#include <inttypes.h>
struct transport_ops;
+struct transport_crypto_ops;
+struct transport_crypto;
typedef struct transport transport_t;
+typedef struct transport_crypto transport_crypto_t;
#include "xlator.h"
#include "dict.h"
@@ -45,9 +48,20 @@ struct transport_msg {
char *hdr;
int hdrlen;
struct iobuf *iobuf;
+ size_t buflen;
+};
+
+struct transport_crypto {
+ struct transport_crypto_ops *ops;
+ void *xl_private;
+ xlator_t *xl;
+ int32_t (*init) (transport_crypto_t *this);
+ void (*fini) (transport_crypto_t *this);
+ uint32_t magic;
};
struct transport {
+ struct transport_crypto crypto;
struct transport_ops *ops;
void *private;
void *xl_private;
@@ -55,6 +69,7 @@ struct transport {
int32_t refcount;
xlator_t *xl;
+
void *dnscache;
data_t *buf;
int32_t (*init) (transport_t *this);
@@ -71,12 +86,18 @@ struct transport {
struct list_head msgs;
struct transport_msg *msg;
} handover;
-
+};
+
+struct transport_crypto_ops {
+ int32_t (*decrypt) (transport_t *this, char *hdr, size_t len,
+ struct iobuf *iobuf_p, size_t buflen);
+ int32_t (*encrypt) (transport_t *this, char *hdr, size_t len,
+ struct iovec *vector, int count);
};
struct transport_ops {
int32_t (*receive) (transport_t *this, char **hdr_p, size_t *hdrlen_p,
- struct iobuf **iobuf_p);
+ struct iobuf **iobuf_p, size_t *buflen_p);
int32_t (*submit) (transport_t *this, char *buf, int len,
struct iovec *vector, int count,
struct iobref *iobref);
diff --git a/transport/Makefile.am b/transport/Makefile.am
index e2f9743..a25d405 100644
--- a/transport/Makefile.am
+++ b/transport/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = socket $(IBVERBS_SUBDIR)
+SUBDIRS = encryption socket $(IBVERBS_SUBDIR)
-CLEANFILES =
+CLEANFILES =
diff --git a/transport/ib-verbs/src/ib-verbs.c b/transport/ib-verbs/src/ib-verbs.c
index 0ec8df8..6fc8172 100644
--- a/transport/ib-verbs/src/ib-verbs.c
+++ b/transport/ib-verbs/src/ib-verbs.c
@@ -549,6 +549,7 @@ ib_verbs_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
}
memcpy (iobuf->ptr, copy_from, size2);
*iobuf_p = iobuf;
+ *buflen_p = size2;
}
err:
@@ -2184,6 +2185,8 @@ ib_verbs_server_event_handler (int fd, int idx, void *data,
this->xl = trans->xl;
this->init = trans->init;
this->fini = trans->fini;
+ memcpy(&this->crypto, &trans->crypto,
+ sizeof (trans->crypto));
memcpy (&this->myinfo.sockaddr, &trans->myinfo.sockaddr,
trans->myinfo.sockaddr_len);
diff --git a/transport/socket/src/socket.c b/transport/socket/src/socket.c
index afd211e..8530f8a 100644
--- a/transport/socket/src/socket.c
+++ b/transport/socket/src/socket.c
@@ -892,6 +892,9 @@ socket_server_event_handler (int fd, int idx, void *data,
new_trans = CALLOC (1, sizeof (*new_trans));
new_trans->xl = this->xl;
new_trans->fini = this->fini;
+ memcpy(&new_trans->crypto, &this->crypto,
+ sizeof (this->crypto));
+ new_trans->crypto.fini = NULL;
memcpy (&new_trans->peerinfo.sockaddr, &new_sockaddr,
addrlen);
@@ -1251,7 +1254,7 @@ unlock:
int
socket_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
- struct iobuf **iobuf_p)
+ struct iobuf **iobuf_p, size_t *buflen_p)
{
socket_private_t *priv = NULL;
int ret = -1;
@@ -1277,6 +1280,7 @@ socket_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
*hdr_p = priv->incoming.hdr_p;
*hdrlen_p = priv->incoming.hdrlen;
*iobuf_p = priv->incoming.iobuf;
+ *buflen_p = priv->incoming.buflen;
memset (&priv->incoming, 0, sizeof (priv->incoming));
priv->incoming.state = SOCKET_PROTO_STATE_NADA;
diff --git a/xlators/protocol/client/src/client-protocol.c b/xlators/protocol/client/src/client-protocol.c
index e82caeb..5e8b1bd 100644
--- a/xlators/protocol/client/src/client-protocol.c
+++ b/xlators/protocol/client/src/client-protocol.c
@@ -6597,5 +6597,8 @@ struct volume_options options[] = {
.min = 5,
.max = 1013,
},
+ { .key = {"transport.crypto.type"},
+ .type = GF_OPTION_TYPE_STR
+ },
{ .key = {NULL} },
};
diff --git a/xlators/protocol/server/src/server-protocol.c b/xlators/protocol/server/src/server-protocol.c
index 3ebebdf..2941788 100644
--- a/xlators/protocol/server/src/server-protocol.c
+++ b/xlators/protocol/server/src/server-protocol.c
@@ -6496,5 +6496,8 @@ struct volume_options options[] = {
{ .key = {"verify-volfile-checksum"},
.type = GF_OPTION_TYPE_BOOL
},
+ { .key = {"transport.crypto.type"},
+ .type = GF_OPTION_TYPE_STR
+ },
{ .key = {NULL} },
};
--
1.6.4.4
More information about the Gluster-devel
mailing list