diff options
Diffstat (limited to 'net/ceph')
| -rw-r--r-- | net/ceph/Kconfig | 3 | ||||
| -rw-r--r-- | net/ceph/Makefile | 2 | ||||
| -rw-r--r-- | net/ceph/auth.c | 309 | ||||
| -rw-r--r-- | net/ceph/decode.c | 45 | ||||
| -rw-r--r-- | net/ceph/messenger.c | 68 | ||||
| -rw-r--r-- | net/ceph/messenger_v2.c | 3443 | ||||
| -rw-r--r-- | net/ceph/mon_client.c | 115 | ||||
| -rw-r--r-- | net/ceph/osd_client.c | 85 |
8 files changed, 4046 insertions, 24 deletions
diff --git a/net/ceph/Kconfig b/net/ceph/Kconfig index f36f9a3a4e20..c5c4eef3a9ff 100644 --- a/net/ceph/Kconfig +++ b/net/ceph/Kconfig @@ -5,6 +5,9 @@ config CEPH_LIB select LIBCRC32C select CRYPTO_AES select CRYPTO_CBC + select CRYPTO_GCM + select CRYPTO_HMAC + select CRYPTO_SHA256 select CRYPTO select KEYS default n diff --git a/net/ceph/Makefile b/net/ceph/Makefile index df02bd8d6c7b..8802a0c0155d 100644 --- a/net/ceph/Makefile +++ b/net/ceph/Makefile @@ -15,4 +15,4 @@ libceph-y := ceph_common.o messenger.o msgpool.o buffer.o pagelist.o \ auth_x.o \ ceph_strings.o ceph_hash.o \ pagevec.o snapshot.o string_table.o \ - messenger_v1.o + messenger_v1.o messenger_v2.o diff --git a/net/ceph/auth.c b/net/ceph/auth.c index 4a0f32b32cc6..6b315c8212b1 100644 --- a/net/ceph/auth.c +++ b/net/ceph/auth.c @@ -293,6 +293,39 @@ int ceph_auth_is_authenticated(struct ceph_auth_client *ac) } EXPORT_SYMBOL(ceph_auth_is_authenticated); +int __ceph_auth_get_authorizer(struct ceph_auth_client *ac, + struct ceph_auth_handshake *auth, + int peer_type, bool force_new, + int *proto, int *pref_mode, int *fallb_mode) +{ + int ret; + + mutex_lock(&ac->mutex); + if (force_new && auth->authorizer) { + ceph_auth_destroy_authorizer(auth->authorizer); + auth->authorizer = NULL; + } + if (!auth->authorizer) + ret = ac->ops->create_authorizer(ac, peer_type, auth); + else if (ac->ops->update_authorizer) + ret = ac->ops->update_authorizer(ac, peer_type, auth); + else + ret = 0; + if (ret) + goto out; + + *proto = ac->protocol; + if (pref_mode && fallb_mode) { + *pref_mode = ac->preferred_mode; + *fallb_mode = ac->fallback_mode; + } + +out: + mutex_unlock(&ac->mutex); + return ret; +} +EXPORT_SYMBOL(__ceph_auth_get_authorizer); + int ceph_auth_create_authorizer(struct ceph_auth_client *ac, int peer_type, struct ceph_auth_handshake *auth) @@ -369,3 +402,279 @@ void ceph_auth_invalidate_authorizer(struct ceph_auth_client *ac, int peer_type) mutex_unlock(&ac->mutex); } EXPORT_SYMBOL(ceph_auth_invalidate_authorizer); + +/* + * msgr2 authentication + */ + +static bool contains(const int *arr, int cnt, int val) +{ + int i; + + for (i = 0; i < cnt; i++) { + if (arr[i] == val) + return true; + } + + return false; +} + +static int encode_con_modes(void **p, void *end, int pref_mode, int fallb_mode) +{ + WARN_ON(pref_mode == CEPH_CON_MODE_UNKNOWN); + if (fallb_mode != CEPH_CON_MODE_UNKNOWN) { + ceph_encode_32_safe(p, end, 2, e_range); + ceph_encode_32_safe(p, end, pref_mode, e_range); + ceph_encode_32_safe(p, end, fallb_mode, e_range); + } else { + ceph_encode_32_safe(p, end, 1, e_range); + ceph_encode_32_safe(p, end, pref_mode, e_range); + } + + return 0; + +e_range: + return -ERANGE; +} + +/* + * Similar to ceph_auth_build_hello(). + */ +int ceph_auth_get_request(struct ceph_auth_client *ac, void *buf, int buf_len) +{ + int proto = ac->key ? CEPH_AUTH_CEPHX : CEPH_AUTH_NONE; + void *end = buf + buf_len; + void *lenp; + void *p; + int ret; + + mutex_lock(&ac->mutex); + if (ac->protocol == CEPH_AUTH_UNKNOWN) { + ret = init_protocol(ac, proto); + if (ret) { + pr_err("auth protocol '%s' init failed: %d\n", + ceph_auth_proto_name(proto), ret); + goto out; + } + } else { + WARN_ON(ac->protocol != proto); + ac->ops->reset(ac); + } + + p = buf; + ceph_encode_32_safe(&p, end, ac->protocol, e_range); + ret = encode_con_modes(&p, end, ac->preferred_mode, ac->fallback_mode); + if (ret) + goto out; + + lenp = p; + p += 4; /* space for len */ + + ceph_encode_8_safe(&p, end, CEPH_AUTH_MODE_MON, e_range); + ret = ceph_auth_entity_name_encode(ac->name, &p, end); + if (ret) + goto out; + + ceph_encode_64_safe(&p, end, ac->global_id, e_range); + ceph_encode_32(&lenp, p - lenp - 4); + ret = p - buf; + +out: + mutex_unlock(&ac->mutex); + return ret; + +e_range: + ret = -ERANGE; + goto out; +} + +int ceph_auth_handle_reply_more(struct ceph_auth_client *ac, void *reply, + int reply_len, void *buf, int buf_len) +{ + int ret; + + mutex_lock(&ac->mutex); + ret = ac->ops->handle_reply(ac, 0, reply, reply + reply_len, + NULL, NULL, NULL, NULL); + if (ret == -EAGAIN) + ret = build_request(ac, false, buf, buf_len); + else + WARN_ON(ret >= 0); + mutex_unlock(&ac->mutex); + return ret; +} + +int ceph_auth_handle_reply_done(struct ceph_auth_client *ac, + u64 global_id, void *reply, int reply_len, + u8 *session_key, int *session_key_len, + u8 *con_secret, int *con_secret_len) +{ + int ret; + + mutex_lock(&ac->mutex); + if (global_id && ac->global_id != global_id) { + dout("%s global_id %llu -> %llu\n", __func__, ac->global_id, + global_id); + ac->global_id = global_id; + } + + ret = ac->ops->handle_reply(ac, 0, reply, reply + reply_len, + session_key, session_key_len, + con_secret, con_secret_len); + mutex_unlock(&ac->mutex); + return ret; +} + +bool ceph_auth_handle_bad_method(struct ceph_auth_client *ac, + int used_proto, int result, + const int *allowed_protos, int proto_cnt, + const int *allowed_modes, int mode_cnt) +{ + mutex_lock(&ac->mutex); + WARN_ON(used_proto != ac->protocol); + + if (result == -EOPNOTSUPP) { + if (!contains(allowed_protos, proto_cnt, ac->protocol)) { + pr_err("auth protocol '%s' not allowed\n", + ceph_auth_proto_name(ac->protocol)); + goto not_allowed; + } + if (!contains(allowed_modes, mode_cnt, ac->preferred_mode) && + (ac->fallback_mode == CEPH_CON_MODE_UNKNOWN || + !contains(allowed_modes, mode_cnt, ac->fallback_mode))) { + pr_err("preferred mode '%s' not allowed\n", + ceph_con_mode_name(ac->preferred_mode)); + if (ac->fallback_mode == CEPH_CON_MODE_UNKNOWN) + pr_err("no fallback mode\n"); + else + pr_err("fallback mode '%s' not allowed\n", + ceph_con_mode_name(ac->fallback_mode)); + goto not_allowed; + } + } + + WARN_ON(result == -EOPNOTSUPP || result >= 0); + pr_err("auth protocol '%s' msgr authentication failed: %d\n", + ceph_auth_proto_name(ac->protocol), result); + + mutex_unlock(&ac->mutex); + return true; + +not_allowed: + mutex_unlock(&ac->mutex); + return false; +} + +int ceph_auth_get_authorizer(struct ceph_auth_client *ac, + struct ceph_auth_handshake *auth, + int peer_type, void *buf, int *buf_len) +{ + void *end = buf + *buf_len; + int pref_mode, fallb_mode; + int proto; + void *p; + int ret; + + ret = __ceph_auth_get_authorizer(ac, auth, peer_type, true, &proto, + &pref_mode, &fallb_mode); + if (ret) + return ret; + + p = buf; + ceph_encode_32_safe(&p, end, proto, e_range); + ret = encode_con_modes(&p, end, pref_mode, fallb_mode); + if (ret) + return ret; + + ceph_encode_32_safe(&p, end, auth->authorizer_buf_len, e_range); + *buf_len = p - buf; + return 0; + +e_range: + return -ERANGE; +} +EXPORT_SYMBOL(ceph_auth_get_authorizer); + +int ceph_auth_handle_svc_reply_more(struct ceph_auth_client *ac, + struct ceph_auth_handshake *auth, + void *reply, int reply_len, + void *buf, int *buf_len) +{ + void *end = buf + *buf_len; + void *p; + int ret; + + ret = ceph_auth_add_authorizer_challenge(ac, auth->authorizer, + reply, reply_len); + if (ret) + return ret; + + p = buf; + ceph_encode_32_safe(&p, end, auth->authorizer_buf_len, e_range); + *buf_len = p - buf; + return 0; + +e_range: + return -ERANGE; +} +EXPORT_SYMBOL(ceph_auth_handle_svc_reply_more); + +int ceph_auth_handle_svc_reply_done(struct ceph_auth_client *ac, + struct ceph_auth_handshake *auth, + void *reply, int reply_len, + u8 *session_key, int *session_key_len, + u8 *con_secret, int *con_secret_len) +{ + return ceph_auth_verify_authorizer_reply(ac, auth->authorizer, + reply, reply_len, session_key, session_key_len, + con_secret, con_secret_len); +} +EXPORT_SYMBOL(ceph_auth_handle_svc_reply_done); + +bool ceph_auth_handle_bad_authorizer(struct ceph_auth_client *ac, + int peer_type, int used_proto, int result, + const int *allowed_protos, int proto_cnt, + const int *allowed_modes, int mode_cnt) +{ + mutex_lock(&ac->mutex); + WARN_ON(used_proto != ac->protocol); + + if (result == -EOPNOTSUPP) { + if (!contains(allowed_protos, proto_cnt, ac->protocol)) { + pr_err("auth protocol '%s' not allowed by %s\n", + ceph_auth_proto_name(ac->protocol), + ceph_entity_type_name(peer_type)); + goto not_allowed; + } + if (!contains(allowed_modes, mode_cnt, ac->preferred_mode) && + (ac->fallback_mode == CEPH_CON_MODE_UNKNOWN || + !contains(allowed_modes, mode_cnt, ac->fallback_mode))) { + pr_err("preferred mode '%s' not allowed by %s\n", + ceph_con_mode_name(ac->preferred_mode), + ceph_entity_type_name(peer_type)); + if (ac->fallback_mode == CEPH_CON_MODE_UNKNOWN) + pr_err("no fallback mode\n"); + else + pr_err("fallback mode '%s' not allowed by %s\n", + ceph_con_mode_name(ac->fallback_mode), + ceph_entity_type_name(peer_type)); + goto not_allowed; + } + } + + WARN_ON(result == -EOPNOTSUPP || result >= 0); + pr_err("auth protocol '%s' authorization to %s failed: %d\n", + ceph_auth_proto_name(ac->protocol), + ceph_entity_type_name(peer_type), result); + + if (ac->ops->invalidate_authorizer) + ac->ops->invalidate_authorizer(ac, peer_type); + + mutex_unlock(&ac->mutex); + return true; + +not_allowed: + mutex_unlock(&ac->mutex); + return false; +} +EXPORT_SYMBOL(ceph_auth_handle_bad_authorizer); diff --git a/net/ceph/decode.c b/net/ceph/decode.c index 6429b6713507..b44f7651be04 100644 --- a/net/ceph/decode.c +++ b/net/ceph/decode.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/ceph/ceph_debug.h> +#include <linux/inet.h> + #include <linux/ceph/decode.h> static int @@ -138,3 +140,46 @@ e_inval: return -EINVAL; } EXPORT_SYMBOL(ceph_decode_entity_addrvec); + +static int get_sockaddr_encoding_len(sa_family_t family) +{ + union { + struct sockaddr sa; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + } u; + + switch (family) { + case AF_INET: + return sizeof(u.sin); + case AF_INET6: + return sizeof(u.sin6); + default: + return sizeof(u); + } +} + +int ceph_entity_addr_encoding_len(const struct ceph_entity_addr *addr) +{ + sa_family_t family = get_unaligned(&addr->in_addr.ss_family); + int addr_len = get_sockaddr_encoding_len(family); + + return 1 + CEPH_ENCODING_START_BLK_LEN + 4 + 4 + 4 + addr_len; +} + +void ceph_encode_entity_addr(void **p, const struct ceph_entity_addr *addr) +{ + sa_family_t family = get_unaligned(&addr->in_addr.ss_family); + int addr_len = get_sockaddr_encoding_len(family); + + ceph_encode_8(p, 1); /* marker */ + ceph_start_encoding(p, 1, 1, sizeof(addr->type) + + sizeof(addr->nonce) + + sizeof(u32) + addr_len); + ceph_encode_copy(p, &addr->type, sizeof(addr->type)); + ceph_encode_copy(p, &addr->nonce, sizeof(addr->nonce)); + + ceph_encode_32(p, addr_len); + ceph_encode_16(p, family); + ceph_encode_copy(p, addr->in_addr.__data, addr_len - sizeof(family)); +} diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 4fb3c33a7b03..57d043b382ed 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -195,8 +195,11 @@ EXPORT_SYMBOL(ceph_pr_addr); void ceph_encode_my_addr(struct ceph_messenger *msgr) { - memcpy(&msgr->my_enc_addr, &msgr->inst.addr, sizeof(msgr->my_enc_addr)); - ceph_encode_banner_addr(&msgr->my_enc_addr); + if (!ceph_msgr2(from_msgr(msgr))) { + memcpy(&msgr->my_enc_addr, &msgr->inst.addr, + sizeof(msgr->my_enc_addr)); + ceph_encode_banner_addr(&msgr->my_enc_addr); + } } /* @@ -513,7 +516,10 @@ static void ceph_con_reset_protocol(struct ceph_connection *con) con->out_msg = NULL; } - ceph_con_v1_reset_protocol(con); + if (ceph_msgr2(from_msgr(con->msgr))) + ceph_con_v2_reset_protocol(con); + else + ceph_con_v1_reset_protocol(con); } /* @@ -526,6 +532,7 @@ static void ceph_msg_remove(struct ceph_msg *msg) ceph_msg_put(msg); } + static void ceph_msg_remove_list(struct list_head *head) { while (!list_empty(head)) { @@ -547,7 +554,10 @@ void ceph_con_reset_session(struct ceph_connection *con) con->in_seq = 0; con->in_seq_acked = 0; - ceph_con_v1_reset_session(con); + if (ceph_msgr2(from_msgr(con->msgr))) + ceph_con_v2_reset_session(con); + else + ceph_con_v1_reset_session(con); } /* @@ -600,6 +610,9 @@ EXPORT_SYMBOL(ceph_con_open); */ bool ceph_con_opened(struct ceph_connection *con) { + if (ceph_msgr2(from_msgr(con->msgr))) + return ceph_con_v2_opened(con); + return ceph_con_v1_opened(con); } @@ -1302,7 +1315,16 @@ int ceph_parse_ips(const char *c, const char *end, } ceph_addr_set_port(&addr[i], port); + /* + * We want the type to be set according to ms_mode + * option, but options are normally parsed after mon + * addresses. Rather than complicating parsing, set + * to LEGACY and override in build_initial_monmap() + * for mon addresses and ceph_messenger_init() for + * ip option. + */ addr[i].type = CEPH_ENTITY_ADDR_TYPE_LEGACY; + addr[i].nonce = 0; dout("parse_ips got %s\n", ceph_pr_addr(&addr[i])); @@ -1410,6 +1432,13 @@ static bool con_sock_closed(struct ceph_connection *con) CASE(PREOPEN); CASE(V1_BANNER); CASE(V1_CONNECT_MSG); + CASE(V2_BANNER_PREFIX); + CASE(V2_BANNER_PAYLOAD); + CASE(V2_HELLO); + CASE(V2_AUTH); + CASE(V2_AUTH_SIGNATURE); + CASE(V2_SESSION_CONNECT); + CASE(V2_SESSION_RECONNECT); CASE(OPEN); CASE(STANDBY); default: @@ -1494,7 +1523,10 @@ static void ceph_con_workfn(struct work_struct *work) BUG_ON(con->sock); } - ret = ceph_con_v1_try_read(con); + if (ceph_msgr2(from_msgr(con->msgr))) + ret = ceph_con_v2_try_read(con); + else + ret = ceph_con_v1_try_read(con); if (ret < 0) { if (ret == -EAGAIN) continue; @@ -1504,7 +1536,10 @@ static void ceph_con_workfn(struct work_struct *work) break; } - ret = ceph_con_v1_try_write(con); + if (ceph_msgr2(from_msgr(con->msgr))) + ret = ceph_con_v2_try_write(con); + else + ret = ceph_con_v1_try_write(con); if (ret < 0) { if (ret == -EAGAIN) continue; @@ -1538,9 +1573,8 @@ static void con_fault(struct ceph_connection *con) ceph_pr_addr(&con->peer_addr), con->error_msg); con->error_msg = NULL; - WARN_ON(con->state != CEPH_CON_S_V1_BANNER && - con->state != CEPH_CON_S_V1_CONNECT_MSG && - con->state != CEPH_CON_S_OPEN); + WARN_ON(con->state == CEPH_CON_S_STANDBY || + con->state == CEPH_CON_S_CLOSED); ceph_con_reset_protocol(con); @@ -1596,7 +1630,11 @@ void ceph_messenger_init(struct ceph_messenger *msgr, ceph_addr_set_port(&msgr->inst.addr, 0); } - msgr->inst.addr.type = 0; + /* + * Since nautilus, clients are identified using type ANY. + * For msgr1, ceph_encode_banner_addr() munges it to NONE. + */ + msgr->inst.addr.type = CEPH_ENTITY_ADDR_TYPE_ANY; /* generate a random non-zero nonce */ do { @@ -1706,7 +1744,10 @@ void ceph_msg_revoke(struct ceph_msg *msg) if (con->out_msg == msg) { WARN_ON(con->state != CEPH_CON_S_OPEN); dout("%s con %p msg %p was sending\n", __func__, con, msg); - ceph_con_v1_revoke(con); + if (ceph_msgr2(from_msgr(con->msgr))) + ceph_con_v2_revoke(con); + else + ceph_con_v1_revoke(con); ceph_msg_put(con->out_msg); con->out_msg = NULL; } else { @@ -1732,7 +1773,10 @@ void ceph_msg_revoke_incoming(struct ceph_msg *msg) if (con->in_msg == msg) { WARN_ON(con->state != CEPH_CON_S_OPEN); dout("%s con %p msg %p was recving\n", __func__, con, msg); - ceph_con_v1_revoke_incoming(con); + if (ceph_msgr2(from_msgr(con->msgr))) + ceph_con_v2_revoke_incoming(con); + else + ceph_con_v1_revoke_incoming(con); ceph_msg_put(con->in_msg); con->in_msg = NULL; } else { diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c new file mode 100644 index 000000000000..5e38c847317b --- /dev/null +++ b/net/ceph/messenger_v2.c @@ -0,0 +1,3443 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ceph msgr2 protocol implementation + * + * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com> + */ + +#include <linux/ceph/ceph_debug.h> + +#include <crypto/aead.h> +#include <crypto/algapi.h> /* for crypto_memneq() */ +#include <crypto/hash.h> +#include <crypto/sha.h> +#include <linux/bvec.h> +#include <linux/crc32c.h> +#include <linux/net.h> +#include <linux/scatterlist.h> +#include <linux/socket.h> +#include <linux/sched/mm.h> +#include <net/sock.h> +#include <net/tcp.h> + +#include <linux/ceph/ceph_features.h> +#include <linux/ceph/decode.h> +#include <linux/ceph/libceph.h> +#include <linux/ceph/messenger.h> + +#include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */ + +#define FRAME_TAG_HELLO 1 +#define FRAME_TAG_AUTH_REQUEST 2 +#define FRAME_TAG_AUTH_BAD_METHOD 3 +#define FRAME_TAG_AUTH_REPLY_MORE 4 +#define FRAME_TAG_AUTH_REQUEST_MORE 5 +#define FRAME_TAG_AUTH_DONE 6 +#define FRAME_TAG_AUTH_SIGNATURE 7 +#define FRAME_TAG_CLIENT_IDENT 8 +#define FRAME_TAG_SERVER_IDENT 9 +#define FRAME_TAG_IDENT_MISSING_FEATURES 10 +#define FRAME_TAG_SESSION_RECONNECT 11 +#define FRAME_TAG_SESSION_RESET 12 +#define FRAME_TAG_SESSION_RETRY 13 +#define FRAME_TAG_SESSION_RETRY_GLOBAL 14 +#define FRAME_TAG_SESSION_RECONNECT_OK 15 +#define FRAME_TAG_WAIT 16 +#define FRAME_TAG_MESSAGE 17 +#define FRAME_TAG_KEEPALIVE2 18 +#define FRAME_TAG_KEEPALIVE2_ACK 19 +#define FRAME_TAG_ACK 20 + +#define FRAME_LATE_STATUS_ABORTED 0x1 +#define FRAME_LATE_STATUS_COMPLETE 0xe +#define FRAME_LATE_STATUS_ABORTED_MASK 0xf + +#define IN_S_HANDLE_PREAMBLE 1 +#define IN_S_HANDLE_CONTROL 2 +#define IN_S_HANDLE_CONTROL_REMAINDER 3 +#define IN_S_PREPARE_READ_DATA 4 +#define IN_S_PREPARE_READ_DATA_CONT 5 +#define IN_S_HANDLE_EPILOGUE 6 +#define IN_S_FINISH_SKIP 7 + +#define OUT_S_QUEUE_DATA 1 +#define OUT_S_QUEUE_DATA_CONT 2 +#define OUT_S_QUEUE_ENC_PAGE 3 +#define OUT_S_QUEUE_ZEROS 4 +#define OUT_S_FINISH_MESSAGE 5 +#define OUT_S_GET_NEXT 6 + +#define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN) +#define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN) +#define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN) +#define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN) + +#define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL) + +static int do_recvmsg(struct socket *sock, struct iov_iter *it) +{ + struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; + int ret; + + msg.msg_iter = *it; + while (iov_iter_count(it)) { + ret = sock_recvmsg(sock, &msg, msg.msg_flags); + if (ret <= 0) { + if (ret == -EAGAIN) + ret = 0; + return ret; + } + + iov_iter_advance(it, ret); + } + + WARN_ON(msg_data_left(&msg)); + return 1; +} + +/* + * Read as much as possible. + * + * Return: + * 1 - done, nothing (else) to read + * 0 - socket is empty, need to wait + * <0 - error + */ +static int ceph_tcp_recv(struct ceph_connection *con) +{ + int ret; + + dout("%s con %p %s %zu\n", __func__, con, + iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need", + iov_iter_count(&con->v2.in_iter)); + ret = do_recvmsg(con->sock, &con->v2.in_iter); + dout("%s con %p ret %d left %zu\n", __func__, con, ret, + iov_iter_count(&con->v2.in_iter)); + return ret; +} + +static int do_sendmsg(struct socket *sock, struct iov_iter *it) +{ + struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; + int ret; + + msg.msg_iter = *it; + while (iov_iter_count(it)) { + ret = sock_sendmsg(sock, &msg); + if (ret <= 0) { + if (ret == -EAGAIN) + ret = 0; + return ret; + } + + iov_iter_advance(it, ret); + } + + WARN_ON(msg_data_left(&msg)); + return 1; +} + +static int do_try_sendpage(struct socket *sock, struct iov_iter *it) +{ + struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; + struct bio_vec bv; + int ret; + + if (WARN_ON(!iov_iter_is_bvec(it))) + return -EINVAL; + + while (iov_iter_count(it)) { + /* iov_iter_iovec() for ITER_BVEC */ + bv.bv_page = it->bvec->bv_page; + bv.bv_offset = it->bvec->bv_offset + it->iov_offset; + bv.bv_len = min(iov_iter_count(it), + it->bvec->bv_len - it->iov_offset); + + /* + * sendpage cannot properly handle pages with + * page_count == 0, we need to fall back to sendmsg if + * that's the case. + * + * Same goes for slab pages: skb_can_coalesce() allows + * coalescing neighboring slab objects into a single frag + * which triggers one of hardened usercopy checks. + */ + if (sendpage_ok(bv.bv_page)) { + ret = sock->ops->sendpage(sock, bv.bv_page, + bv.bv_offset, bv.bv_len, + CEPH_MSG_FLAGS); + } else { + iov_iter_bvec(&msg.msg_iter, WRITE, &bv, 1, bv.bv_len); + ret = sock_sendmsg(sock, &msg); + } + if (ret <= 0) { + if (ret == -EAGAIN) + ret = 0; + return ret; + } + + iov_iter_advance(it, ret); + } + + return 1; +} + +/* + * Write as much as possible. The socket is expected to be corked, + * so we don't bother with MSG_MORE/MSG_SENDPAGE_NOTLAST here. + * + * Return: + * 1 - done, nothing (else) to write + * 0 - socket is full, need to wait + * <0 - error + */ +static int ceph_tcp_send(struct ceph_connection *con) +{ + int ret; + + dout("%s con %p have %zu try_sendpage %d\n", __func__, con, + iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage); + if (con->v2.out_iter_sendpage) + ret = do_try_sendpage(con->sock, &con->v2.out_iter); + else + ret = do_sendmsg(con->sock, &con->v2.out_iter); + dout("%s con %p ret %d left %zu\n", __func__, con, ret, + iov_iter_count(&con->v2.out_iter)); + return ret; +} + +static void add_in_kvec(struct ceph_connection *con, void *buf, int len) +{ + BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs)); + WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); + + con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf; + con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len; + con->v2.in_kvec_cnt++; + + con->v2.in_iter.nr_segs++; + con->v2.in_iter.count += len; +} + +static void reset_in_kvecs(struct ceph_connection *con) +{ + WARN_ON(iov_iter_count(&con->v2.in_iter)); + + con->v2.in_kvec_cnt = 0; + iov_iter_kvec(&con->v2.in_iter, READ, con->v2.in_kvecs, 0, 0); +} + +static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv) +{ + WARN_ON(iov_iter_count(&con->v2.in_iter)); + + con->v2.in_bvec = *bv; + iov_iter_bvec(&con->v2.in_iter, READ, &con->v2.in_bvec, 1, bv->bv_len); +} + +static void set_in_skip(struct ceph_connection *con, int len) +{ + WARN_ON(iov_iter_count(&con->v2.in_iter)); + + dout("%s con %p len %d\n", __func__, con, len); + iov_iter_discard(&con->v2.in_iter, READ, len); +} + +static void add_out_kvec(struct ceph_connection *con, void *buf, int len) +{ + BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs)); + WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); + WARN_ON(con->v2.out_zero); + + con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf; + con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len; + con->v2.out_kvec_cnt++; + + con->v2.out_iter.nr_segs++; + con->v2.out_iter.count += len; +} + +static void reset_out_kvecs(struct ceph_connection *con) +{ + WARN_ON(iov_iter_count(&con->v2.out_iter)); + WARN_ON(con->v2.out_zero); + + con->v2.out_kvec_cnt = 0; + + iov_iter_kvec(&con->v2.out_iter, WRITE, con->v2.out_kvecs, 0, 0); + con->v2.out_iter_sendpage = false; +} + +static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv, + bool zerocopy) +{ + WARN_ON(iov_iter_count(&con->v2.out_iter)); + WARN_ON(con->v2.out_zero); + + con->v2.out_bvec = *bv; + con->v2.out_iter_sendpage = zerocopy; + iov_iter_bvec(&con->v2.out_iter, WRITE, &con->v2.out_bvec, 1, + con->v2.out_bvec.bv_len); +} + +static void set_out_bvec_zero(struct ceph_connection *con) +{ + WARN_ON(iov_iter_count(&con->v2.out_iter)); + WARN_ON(!con->v2.out_zero); + + con->v2.out_bvec.bv_page = ceph_zero_page; + con->v2.out_bvec.bv_offset = 0; + con->v2.out_bvec.bv_len = min(con->v2.out_zero, (int)PAGE_SIZE); + con->v2.out_iter_sendpage = true; + iov_iter_bvec(&con->v2.out_iter, WRITE, &con->v2.out_bvec, 1, + con->v2.out_bvec.bv_len); +} + +static void out_zero_add(struct ceph_connection *con, int len) +{ + dout("%s con %p len %d\n", __func__, con, len); + con->v2.out_zero += len; +} + +static void *alloc_conn_buf(struct ceph_connection *con, int len) +{ + void *buf; + + dout("%s con %p len %d\n", __func__, con, len); + + if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs))) + return NULL; + + buf = ceph_kvmalloc(len, GFP_NOIO); + if (!buf) + return NULL; + + con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf; + return buf; +} + +static void free_conn_bufs(struct ceph_connection *con) +{ + while (con->v2.conn_buf_cnt) + kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]); +} + +static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len) +{ + BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs)); + + con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf; + con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len; + con->v2.in_sign_kvec_cnt++; +} + +static void clear_in_sign_kvecs(struct ceph_connection *con) +{ + con->v2.in_sign_kvec_cnt = 0; +} + +static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len) +{ + BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs)); + + con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf; + con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len; + con->v2.out_sign_kvec_cnt++; +} + +static void clear_out_sign_kvecs(struct ceph_connection *con) +{ + con->v2.out_sign_kvec_cnt = 0; +} + +static bool con_secure(struct ceph_connection *con) +{ + return con->v2.con_mode == CEPH_CON_MODE_SECURE; +} + +static int front_len(const struct ceph_msg *msg) +{ + return le32_to_cpu(msg->hdr.front_len); +} + +static int middle_len(const struct ceph_msg *msg) +{ + return le32_to_cpu(msg->hdr.middle_len); +} + +static int data_len(const struct ceph_msg *msg) +{ + return le32_to_cpu(msg->hdr.data_len); +} + +static bool need_padding(int len) +{ + return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN); +} + +static int padded_len(int len) +{ + return ALIGN(len, CEPH_GCM_BLOCK_LEN); +} + +static int padding_len(int len) +{ + return padded_len(len) - len; +} + +/* preamble + control segment */ +static int head_onwire_len(int ctrl_len, bool secure) +{ + int head_len; + int rem_len; + + if (secure) { + head_len = CEPH_PREAMBLE_SECURE_LEN; + if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) { + rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; + head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN; + } + } else { + head_len = CEPH_PREAMBLE_PLAIN_LEN; + if (ctrl_len) + head_len += ctrl_len + CEPH_CRC_LEN; + } + return head_len; +} + +/* front, middle and data segments + epilogue */ +static int __tail_onwire_len(int front_len, int middle_len, int data_len, + bool secure) +{ + if (!front_len && !middle_len && !data_len) + return 0; + + if (!secure) + return front_len + middle_len + data_len + + CEPH_EPILOGUE_PLAIN_LEN; + + return padded_len(front_len) + padded_len(middle_len) + + padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN; +} + +static int tail_onwire_len(const struct ceph_msg *msg, bool secure) +{ + return __tail_onwire_len(front_len(msg), middle_len(msg), + data_len(msg), secure); +} + +/* head_onwire_len(sizeof(struct ceph_msg_header2), false) */ +#define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \ + sizeof(struct ceph_msg_header2) + \ + CEPH_CRC_LEN) + +static const int frame_aligns[] = { + sizeof(void *), + sizeof(void *), + sizeof(void *), + PAGE_SIZE +}; + +/* + * Discards trailing empty segments, unless there is just one segment. + * A frame always has at least one (possibly empty) segment. + */ +static int calc_segment_count(const int *lens, int len_cnt) +{ + int i; + + for (i = len_cnt - 1; i >= 0; i--) { + if (lens[i]) + return i + 1; + } + + return 1; +} + +static void init_frame_desc(struct ceph_frame_desc *desc, int tag, + const int *lens, int len_cnt) +{ + int i; + + memset(desc, 0, sizeof(*desc)); + + desc->fd_tag = tag; + desc->fd_seg_cnt = calc_segment_count(lens, len_cnt); + BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT); + for (i = 0; i < desc->fd_seg_cnt; i++) { + desc->fd_lens[i] = lens[i]; + desc->fd_aligns[i] = frame_aligns[i]; + } +} + +/* + * Preamble crc covers everything up to itself (28 bytes) and + * is calculated and verified irrespective of the connection mode + * (i.e. even if the frame is encrypted). + */ +static void encode_preamble(const struct ceph_frame_desc *desc, void *p) +{ + void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN; + void *start = p; + int i; + + memset(p, 0, CEPH_PREAMBLE_LEN); + + ceph_encode_8(&p, desc->fd_tag); + ceph_encode_8(&p, desc->fd_seg_cnt); + for (i = 0; i < desc->fd_seg_cnt; i++) { + ceph_encode_32(&p, desc->fd_lens[i]); + ceph_encode_16(&p, desc->fd_aligns[i]); + } + + put_unaligned_le32(crc32c(0, start, crcp - start), crcp); +} + +static int decode_preamble(void *p, struct ceph_frame_desc *desc) +{ + void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN; + u32 crc, expected_crc; + int i; + + crc = crc32c(0, p, crcp - p); + expected_crc = get_unaligned_le32(crcp); + if (crc != expected_crc) { + pr_err("bad preamble crc, calculated %u, expected %u\n", + crc, expected_crc); + return -EBADMSG; + } + + memset(desc, 0, sizeof(*desc)); + + desc->fd_tag = ceph_decode_8(&p); + desc->fd_seg_cnt = ceph_decode_8(&p); + if (desc->fd_seg_cnt < 1 || + desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) { + pr_err("bad segment count %d\n", desc->fd_seg_cnt); + return -EINVAL; + } + for (i = 0; i < desc->fd_seg_cnt; i++) { + desc->fd_lens[i] = ceph_decode_32(&p); + desc->fd_aligns[i] = ceph_decode_16(&p); + } + + /* + * This would fire for FRAME_TAG_WAIT (it has one empty + * segment), but we should never get it as client. + */ + if (!desc->fd_lens[desc->fd_seg_cnt - 1]) { + pr_err("last segment empty\n"); + return -EINVAL; + } + + if (desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) { + pr_err("control segment too big %d\n", desc->fd_lens[0]); + return -EINVAL; + } + if (desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) { + pr_err("front segment too big %d\n", desc->fd_lens[1]); + return -EINVAL; + } + if (desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) { + pr_err("middle segment too big %d\n", desc->fd_lens[2]); + return -EINVAL; + } + if (desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) { + pr_err("data segment too big %d\n", desc->fd_lens[3]); + return -EINVAL; + } + + return 0; +} + +static void encode_epilogue_plain(struct ceph_connection *con, bool aborted) +{ + con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED : + FRAME_LATE_STATUS_COMPLETE; + cpu_to_le32s(&con->v2.out_epil.front_crc); + cpu_to_le32s(&con->v2.out_epil.middle_crc); + cpu_to_le32s(&con->v2.out_epil.data_crc); +} + +static void encode_epilogue_secure(struct ceph_connection *con, bool aborted) +{ + memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil)); + con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED : + FRAME_LATE_STATUS_COMPLETE; +} + +static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc, + u32 *data_crc) +{ + u8 late_status; + + late_status = ceph_decode_8(&p); + if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) != + FRAME_LATE_STATUS_COMPLETE) { + /* we should never get an aborted message as client */ + pr_err("bad late_status 0x%x\n", late_status); + return -EINVAL; + } + + if (front_crc && middle_crc && data_crc) { + *front_crc = ceph_decode_32(&p); + *middle_crc = ceph_decode_32(&p); + *data_crc = ceph_decode_32(&p); + } + + return 0; +} + +static void fill_header(struct ceph_msg_header *hdr, + const struct ceph_msg_header2 *hdr2, + int front_len, int middle_len, int data_len, + const struct ceph_entity_name *peer_name) +{ + hdr->seq = hdr2->seq; + hdr->tid = hdr2->tid; + hdr->type = hdr2->type; + hdr->priority = hdr2->priority; + hdr->version = hdr2->version; + hdr->front_len = cpu_to_le32(front_len); + hdr->middle_len = cpu_to_le32(middle_len); + hdr->data_len = cpu_to_le32(data_len); + hdr->data_off = hdr2->data_off; + hdr->src = *peer_name; + hdr->compat_version = hdr2->compat_version; + hdr->reserved = 0; + hdr->crc = 0; +} + +static void fill_header2(struct ceph_msg_header2 *hdr2, + const struct ceph_msg_header *hdr, u64 ack_seq) +{ + hdr2->seq = hdr->seq; + hdr2->tid = hdr->tid; + hdr2->type = hdr->type; + hdr2->priority = hdr->priority; + hdr2->version = hdr->version; + hdr2->data_pre_padding_len = 0; + hdr2->data_off = hdr->data_off; + hdr2->ack_seq = cpu_to_le64(ack_seq); + hdr2->flags = 0; + hdr2->compat_version = hdr->compat_version; + hdr2->reserved = 0; +} + +static int verify_control_crc(struct ceph_connection *con) +{ + int ctrl_len = con->v2.in_desc.fd_lens[0]; + u32 crc, expected_crc; + + WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len); + WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN); + + crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len); + expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base); + if (crc != expected_crc) { + pr_err("bad control crc, calculated %u, expected %u\n", + crc, expected_crc); + return -EBADMSG; + } + + return 0; +} + +static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc, + u32 middle_crc, u32 data_crc) +{ + if (front_len(con->in_msg)) { + con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base, + front_len(con->in_msg)); + } else { + WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg)); + con->in_front_crc = -1; + } + + if (middle_len(con->in_msg)) + con->in_middle_crc = crc32c(-1, + con->in_msg->middle->vec.iov_base, + middle_len(con->in_msg)); + else if (data_len(con->in_msg)) + con->in_middle_crc = -1; + else + con->in_middle_crc = 0; + + if (!data_len(con->in_msg)) + con->in_data_crc = 0; + + dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg, + con->in_front_crc, con->in_middle_crc, con->in_data_crc); + + if (con->in_front_crc != front_crc) { + pr_err("bad front crc, calculated %u, expected %u\n", + con->in_front_crc, front_crc); + return -EBADMSG; + } + if (con->in_middle_crc != middle_crc) { + pr_err("bad middle crc, calculated %u, expected %u\n", + con->in_middle_crc, middle_crc); + return -EBADMSG; + } + if (con->in_data_crc != data_crc) { + pr_err("bad data crc, calculated %u, expected %u\n", + con->in_data_crc, data_crc); + return -EBADMSG; + } + + return 0; +} + +static int setup_crypto(struct ceph_connection *con, + u8 *session_key, int session_key_len, + u8 *con_secret, int con_secret_len) +{ + unsigned int noio_flag; + void *p; + int ret; + + dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n", + __func__, con, con->v2.con_mode, session_key_len, con_secret_len); + WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req); + + if (con->v2.con_mode != CEPH_CON_MODE_CRC && + con->v2.con_mode != CEPH_CON_MODE_SECURE) { + pr_err("bad con_mode %d\n", con->v2.con_mode); + return -EINVAL; + } + + if (!session_key_len) { + WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC); + WARN_ON(con_secret_len); + return 0; /* auth_none */ + } + + noio_flag = memalloc_noio_save(); + con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); + memalloc_noio_restore(noio_flag); + if (IS_ERR(con->v2.hmac_tfm)) { + ret = PTR_ERR(con->v2.hmac_tfm); + con->v2.hmac_tfm = NULL; + pr_err("failed to allocate hmac tfm context: %d\n", ret); + return ret; + } + + WARN_ON((unsigned long)session_key & + crypto_shash_alignmask(con->v2.hmac_tfm)); + ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key, + session_key_len); + if (ret) { + pr_err("failed to set hmac key: %d\n", ret); + return ret; + } + + if (con->v2.con_mode == CEPH_CON_MODE_CRC) { + WARN_ON(con_secret_len); + return 0; /* auth_x, plain mode */ + } + + if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) { + pr_err("con_secret too small %d\n", con_secret_len); + return -EINVAL; + } + + noio_flag = memalloc_noio_save(); + con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + memalloc_noio_restore(noio_flag); + if (IS_ERR(con->v2.gcm_tfm)) { + ret = PTR_ERR(con->v2.gcm_tfm); + con->v2.gcm_tfm = NULL; + pr_err("failed to allocate gcm tfm context: %d\n", ret); + return ret; + } + + p = con_secret; + WARN_ON((unsigned long)p & crypto_aead_alignmask(con->v2.gcm_tfm)); + ret = crypto_aead_setkey(con->v2.gcm_tfm, p, CEPH_GCM_KEY_LEN); + if (ret) { + pr_err("failed to set gcm key: %d\n", ret); + return ret; + } + + p += CEPH_GCM_KEY_LEN; + WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN); + ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN); + if (ret) { + pr_err("failed to set gcm tag size: %d\n", ret); + return ret; + } + + con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO); + if (!con->v2.gcm_req) { + pr_err("failed to allocate gcm request\n"); + return -ENOMEM; + } + + crypto_init_wait(&con->v2.gcm_wait); + aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &con->v2.gcm_wait); + + memcpy(&con->v2.in_gcm_nonce, p, CEPH_GCM_IV_LEN); + memcpy(&con->v2.out_gcm_nonce, p + CEPH_GCM_IV_LEN, CEPH_GCM_IV_LEN); + return 0; /* auth_x, secure mode */ +} + +static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs, + int kvec_cnt, u8 *hmac) +{ + SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */ + int ret; + int i; + + dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con, + con->v2.hmac_tfm, kvec_cnt); + + if (!con->v2.hmac_tfm) { + memset(hmac, 0, SHA256_DIGEST_SIZE); + return 0; /* auth_none */ + } + + desc->tfm = con->v2.hmac_tfm; + ret = crypto_shash_init(desc); + if (ret) + return ret; + + for (i = 0; i < kvec_cnt; i++) { + WARN_ON((unsigned long)kvecs[i].iov_base & + crypto_shash_alignmask(con->v2.hmac_tfm)); + ret = crypto_shash_update(desc, kvecs[i].iov_base, + kvecs[i].iov_len); + if (ret) + return ret; + } + + ret = crypto_shash_final(desc, hmac); + if (ret) + return ret; + + shash_desc_zero(desc); + return 0; /* auth_x, both plain and secure modes */ +} + +static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce) +{ + u64 counter; + + counter = le64_to_cpu(nonce->counter); + nonce->counter = cpu_to_le64(counter + 1); +} + +static int gcm_crypt(struct ceph_connection *con, bool encrypt, + struct scatterlist *src, struct scatterlist *dst, + int src_len) +{ + struct ceph_gcm_nonce *nonce; + int ret; + + nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce; + + aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */ + aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce); + ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) : + crypto_aead_decrypt(con->v2.gcm_req), + &con->v2.gcm_wait); + if (ret) + return ret; + + gcm_inc_nonce(nonce); + return 0; +} + +static void get_bvec_at(struct ceph_msg_data_cursor *cursor, + struct bio_vec *bv) +{ + struct page *page; + size_t off, len; + + WARN_ON(!cursor->total_resid); + + /* skip zero-length data items */ + while (!cursor->resid) + ceph_msg_data_advance(cursor, 0); + + /* get a piece of data, cursor isn't advanced */ + page = ceph_msg_data_next(cursor, &off, &len, NULL); + + bv->bv_page = page; + bv->bv_offset = off; + bv->bv_len = len; +} + +static int calc_sg_cnt(void *buf, int buf_len) +{ + int sg_cnt; + + if (!buf_len) + return 0; + + sg_cnt = need_padding(buf_len) ? 1 : 0; + if (is_vmalloc_addr(buf)) { + WARN_ON(offset_in_page(buf)); + sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT; + } else { + sg_cnt++; + } + + return sg_cnt; +} + +static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor) +{ + int data_len = cursor->total_resid; + struct bio_vec bv; + int sg_cnt; + + if (!data_len) + return 0; + + sg_cnt = need_padding(data_len) ? 1 : 0; + do { + get_bvec_at(cursor, &bv); + sg_cnt++; + + ceph_msg_data_advance(cursor, bv.bv_len); + } while (cursor->total_resid); + + return sg_cnt; +} + +static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad) +{ + void *end = buf + buf_len; + struct page *page; + int len; + void *p; + + if (!buf_len) + return; + + if (is_vmalloc_addr(buf)) { + p = buf; + do { + page = vmalloc_to_page(p); + len = min_t(int, end - p, PAGE_SIZE); + WARN_ON(!page || !len || offset_in_page(p)); + sg_set_page(*sg, page, len, 0); + *sg = sg_next(*sg); + p += len; + } while (p != end); + } else { + sg_set_buf(*sg, buf, buf_len); + *sg = sg_next(*sg); + } + + if (need_padding(buf_len)) { + sg_set_buf(*sg, pad, padding_len(buf_len)); + *sg = sg_next(*sg); + } +} + +static void init_sgs_cursor(struct scatterlist **sg, + struct ceph_msg_data_cursor *cursor, u8 *pad) +{ + int data_len = cursor->total_resid; + struct bio_vec bv; + + if (!data_len) + return; + + do { + get_bvec_at(cursor, &bv); + sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); + *sg = sg_next(*sg); + + ceph_msg_data_advance(cursor, bv.bv_len); + } while (cursor->total_resid); + + if (need_padding(data_len)) { + sg_set_buf(*sg, pad, padding_len(data_len)); + *sg = sg_next(*sg); + } +} + +static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg, + u8 *front_pad, u8 *middle_pad, u8 *data_pad, + void *epilogue, bool add_tag) +{ + struct ceph_msg_data_cursor cursor; + struct scatterlist *cur_sg; + int sg_cnt; + int ret; + + if (!front_len(msg) && !middle_len(msg) && !data_len(msg)) + return 0; + + sg_cnt = 1; /* epilogue + [auth tag] */ + if (front_len(msg)) + sg_cnt += calc_sg_cnt(msg->front.iov_base, + front_len(msg)); + if (middle_len(msg)) + sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base, + middle_len(msg)); + if (data_len(msg)) { + ceph_msg_data_cursor_init(&cursor, msg, data_len(msg)); + sg_cnt += calc_sg_cnt_cursor(&cursor); + } + + ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO); + if (ret) + return ret; + + cur_sg = sgt->sgl; + if (front_len(msg)) + init_sgs(&cur_sg, msg->front.iov_base, front_len(msg), + front_pad); + if (middle_len(msg)) + init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg), + middle_pad); + if (data_len(msg)) { + ceph_msg_data_cursor_init(&cursor, msg, data_len(msg)); + init_sgs_cursor(&cur_sg, &cursor, data_pad); + } + + WARN_ON(!sg_is_last(cur_sg)); + sg_set_buf(cur_sg, epilogue, + CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0)); + return 0; +} + +static int decrypt_preamble(struct ceph_connection *con) +{ + struct scatterlist sg; + + sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN); + return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN); +} + +static int decrypt_control_remainder(struct ceph_connection *con) +{ + int ctrl_len = con->v2.in_desc.fd_lens[0]; + int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; + int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN; + struct scatterlist sgs[2]; + + WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len); + WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len); + + sg_init_table(sgs, 2); + sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len); + sg_set_buf(&sgs[1], con->v2.in_buf, pt_len); + + return gcm_crypt(con, false, sgs, sgs, + padded_len(rem_len) + CEPH_GCM_TAG_LEN); +} + +static int decrypt_message(struct ceph_connection *con) +{ + struct sg_table sgt = {}; + int ret; + + ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf), + MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf), + con->v2.in_buf, true); + if (ret) + goto out; + + ret = gcm_crypt(con, false, sgt.sgl, sgt.sgl, + tail_onwire_len(con->in_msg, true)); + +out: + sg_free_table(&sgt); + return ret; +} + +static int prepare_banner(struct ceph_connection *con) +{ + int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8; + void *buf, *p; + + buf = alloc_conn_buf(con, buf_len); + if (!buf) + return -ENOMEM; + + p = buf; + ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN); + ceph_encode_16(&p, sizeof(u64) + sizeof(u64)); + ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES); + ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES); + WARN_ON(p != buf + buf_len); + + add_out_kvec(con, buf, buf_len); + add_out_sign_kvec(con, buf, buf_len); + ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); + return 0; +} + +/* + * base: + * preamble + * control body (ctrl_len bytes) + * space for control crc + * + * extdata (optional): + * control body (extdata_len bytes) + * + * Compute control crc and gather base and extdata into: + * + * preamble + * control body (ctrl_len + extdata_len bytes) + * control crc + * + * Preamble should already be encoded at the start of base. + */ +static void prepare_head_plain(struct ceph_connection *con, void *base, + int ctrl_len, void *extdata, int extdata_len, + bool to_be_signed) +{ + int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN; + void *crcp = base + base_len - CEPH_CRC_LEN; + u32 crc; + + crc = crc32c(-1, CTRL_BODY(base), ctrl_len); + if (extdata_len) + crc = crc32c(crc, extdata, extdata_len); + put_unaligned_le32(crc, crcp); + + if (!extdata_len) { + add_out_kvec(con, base, base_len); + if (to_be_signed) + add_out_sign_kvec(con, base, base_len); + return; + } + + add_out_kvec(con, base, crcp - base); + add_out_kvec(con, extdata, extdata_len); + add_out_kvec(con, crcp, CEPH_CRC_LEN); + if (to_be_signed) { + add_out_sign_kvec(con, base, crcp - base); + add_out_sign_kvec(con, extdata, extdata_len); + add_out_sign_kvec(con, crcp, CEPH_CRC_LEN); + } +} + +static int prepare_head_secure_small(struct ceph_connection *con, + void *base, int ctrl_len) +{ + struct scatterlist sg; + int ret; + + /* inline buffer padding? */ + if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN) + memset(CTRL_BODY(base) + ctrl_len, 0, + CEPH_PREAMBLE_INLINE_LEN - ctrl_len); + + sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN); + ret = gcm_crypt(con, true, &sg, &sg, + CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN); + if (ret) + return ret; + + add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN); + return 0; +} + +/* + * base: + * preamble + * control body (ctrl_len bytes) + * space for padding, if needed + * space for control remainder auth tag + * space for preamble auth tag + * + * Encrypt preamble and the inline portion, then encrypt the remainder + * and gather into: + * + * preamble + * control body (48 bytes) + * preamble auth tag + * control body (ctrl_len - 48 bytes) + * zero padding, if needed + * control remainder auth tag + * + * Preamble should already be encoded at the start of base. + */ +static int prepare_head_secure_big(struct ceph_connection *con, + void *base, int ctrl_len) +{ + int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; + void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN; + void *rem_tag = rem + padded_len(rem_len); + void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN; + struct scatterlist sgs[2]; + int ret; + + sg_init_table(sgs, 2); + sg_set_buf(&sgs[0], base, rem - base); + sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN); + ret = gcm_crypt(con, true, sgs, sgs, rem - base); + if (ret) + return ret; + + /* control remainder padding? */ + if (need_padding(rem_len)) + memset(rem + rem_len, 0, padding_len(rem_len)); + + sg_init_one(&sgs[0], rem, pmbl_tag - rem); + ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem); + if (ret) + return ret; + + add_out_kvec(con, base, rem - base); + add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN); + add_out_kvec(con, rem, pmbl_tag - rem); + return 0; +} + +static int __prepare_control(struct ceph_connection *con, int tag, + void *base, int ctrl_len, void *extdata, + int extdata_len, bool to_be_signed) +{ + int total_len = ctrl_len + extdata_len; + struct ceph_frame_desc desc; + int ret; + + dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag, + total_len, ctrl_len, extdata_len); + + /* extdata may be vmalloc'ed but not base */ + if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len)) + return -EINVAL; + + init_frame_desc(&desc, tag, &total_len, 1); + encode_preamble(&desc, base); + + if (con_secure(con)) { + if (WARN_ON(extdata_len || to_be_signed)) + return -EINVAL; + + if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN) + /* fully inlined, inline buffer may need padding */ + ret = prepare_head_secure_small(con, base, ctrl_len); + else + /* partially inlined, inline buffer is full */ + ret = prepare_head_secure_big(con, base, ctrl_len); + if (ret) + return ret; + } else { + prepare_head_plain(con, base, ctrl_len, extdata, extdata_len, + to_be_signed); + } + + ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); + return 0; +} + +static int prepare_control(struct ceph_connection *con, int tag, + void *base, int ctrl_len) +{ + return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false); +} + +static int prepare_hello(struct ceph_connection *con) +{ + void *buf, *p; + int ctrl_len; + + ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr); + buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); + if (!buf) + return -ENOMEM; + + p = CTRL_BODY(buf); + ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT); + ceph_encode_entity_addr(&p, &con->peer_addr); + WARN_ON(p != CTRL_BODY(buf) + ctrl_len); + + return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len, + NULL, 0, true); +} + +/* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */ +#define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN) + +static int prepare_auth_request(struct ceph_connection *con) +{ + void *authorizer, *authorizer_copy; + int ctrl_len, authorizer_len; + void *buf; + int ret; + + ctrl_len = AUTH_BUF_LEN; + buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); + if (!buf) + return -ENOMEM; + + mutex_unlock(&con->mutex); + ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len, + &authorizer, &authorizer_len); + mutex_lock(&con->mutex); + if (con->state != CEPH_CON_S_V2_HELLO) { + dout("%s con %p state changed to %d\n", __func__, con, + con->state); + return -EAGAIN; + } + + dout("%s con %p get_auth_request ret %d\n", __func__, con, ret); + if (ret) + return ret; + + authorizer_copy = alloc_conn_buf(con, authorizer_len); + if (!authorizer_copy) + return -ENOMEM; + + memcpy(authorizer_copy, authorizer, authorizer_len); + + return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len, + authorizer_copy, authorizer_len, true); +} + +static int prepare_auth_request_more(struct ceph_connection *con, + void *reply, int reply_len) +{ + int ctrl_len, authorizer_len; + void *authorizer; + void *buf; + int ret; + + ctrl_len = AUTH_BUF_LEN; + buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); + if (!buf) + return -ENOMEM; + + mutex_unlock(&con->mutex); + ret = con->ops->handle_auth_reply_more(con, reply, reply_len, + CTRL_BODY(buf), &ctrl_len, + &authorizer, &authorizer_len); + mutex_lock(&con->mutex); + if (con->state != CEPH_CON_S_V2_AUTH) { + dout("%s con %p state changed to %d\n", __func__, con, + con->state); + return -EAGAIN; + } + + dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret); + if (ret) + return ret; + + return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf, + ctrl_len, authorizer, authorizer_len, true); +} + +static int prepare_auth_signature(struct ceph_connection *con) +{ + void *buf; + int ret; + + buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE, false)); + if (!buf) + return -ENOMEM; + + ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt, + CTRL_BODY(buf)); + if (ret) + return ret; + + return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf, + SHA256_DIGEST_SIZE); +} + +static int prepare_client_ident(struct ceph_connection *con) +{ + struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; + struct ceph_client *client = from_msgr(con->msgr); + u64 global_id = ceph_client_gid(client); + void *buf, *p; + int ctrl_len; + + WARN_ON(con->v2.server_cookie); + WARN_ON(con->v2.connect_seq); + WARN_ON(con->v2.peer_global_seq); + + if (!con->v2.client_cookie) { + do { + get_random_bytes(&con->v2.client_cookie, + sizeof(con->v2.client_cookie)); + } while (!con->v2.client_cookie); + dout("%s con %p generated cookie 0x%llx\n", __func__, con, + con->v2.client_cookie); + } else { + dout("%s con %p cookie already set 0x%llx\n", __func__, con, + con->v2.client_cookie); + } + + dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n", + __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce), + ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce), + global_id, con->v2.global_seq, client->supported_features, + client->required_features, con->v2.client_cookie); + + ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + + ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8; + buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con))); + if (!buf) + return -ENOMEM; + + p = CTRL_BODY(buf); + ceph_encode_8(&p, 2); /* addrvec marker */ + ceph_encode_32(&p, 1); /* addr_cnt */ + ceph_encode_entity_addr(&p, my_addr); + ceph_encode_entity_addr(&p, &con->peer_addr); + ceph_encode_64(&p, global_id); + ceph_encode_64(&p, con->v2.global_seq); + ceph_encode_64(&p, client->supported_features); + ceph_encode_64(&p, client->required_features); + ceph_encode_64(&p, 0); /* flags */ + ceph_encode_64(&p, con->v2.client_cookie); + WARN_ON(p != CTRL_BODY(buf) + ctrl_len); + + return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len); +} + +static int prepare_session_reconnect(struct ceph_connection *con) +{ + struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; + void *buf, *p; + int ctrl_len; + + WARN_ON(!con->v2.client_cookie); + WARN_ON(!con->v2.server_cookie); + WARN_ON(!con->v2.connect_seq); + WARN_ON(!con->v2.peer_global_seq); + + dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n", + __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce), + con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq, + con->v2.connect_seq, con->in_seq); + + ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8; + buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con))); + if (!buf) + return -ENOMEM; + + p = CTRL_BODY(buf); + ceph_encode_8(&p, 2); /* entity_addrvec_t marker */ + ceph_encode_32(&p, 1); /* my_addrs len */ + ceph_encode_entity_addr(&p, my_addr); + ceph_encode_64(&p, con->v2.client_cookie); + ceph_encode_64(&p, con->v2.server_cookie); + ceph_encode_64(&p, con->v2.global_seq); + ceph_encode_64(&p, con->v2.connect_seq); + ceph_encode_64(&p, con->in_seq); + WARN_ON(p != CTRL_BODY(buf) + ctrl_len); + + return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len); +} + +static int prepare_keepalive2(struct ceph_connection *con) +{ + struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf); + struct timespec64 now; + + ktime_get_real_ts64(&now); + dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec, + now.tv_nsec); + + ceph_encode_timespec64(ts, &now); + + reset_out_kvecs(con); + return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf, + sizeof(struct ceph_timespec)); +} + +static int prepare_ack(struct ceph_connection *con) +{ + void *p; + + dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con, + con->in_seq_acked, con->in_seq); + con->in_seq_acked = con->in_seq; + + p = CTRL_BODY(con->v2.out_buf); + ceph_encode_64(&p, con->in_seq_acked); + + reset_out_kvecs(con); + return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8); +} + +static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted) +{ + dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con, + con->out_msg, aborted, con->v2.out_epil.front_crc, + con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc); + + encode_epilogue_plain(con, aborted); + add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN); +} + +/* + * For "used" empty segments, crc is -1. For unused (trailing) + * segments, crc is 0. + */ +static void prepare_message_plain(struct ceph_connection *con) +{ + struct ceph_msg *msg = con->out_msg; + + prepare_head_plain(con, con->v2.out_buf, + sizeof(struct ceph_msg_header2), NULL, 0, false); + + if (!front_len(msg) && !middle_len(msg)) { + if (!data_len(msg)) { + /* + * Empty message: once the head is written, + * we are done -- there is no epilogue. + */ + con->v2.out_state = OUT_S_FINISH_MESSAGE; + return; + } + + con->v2.out_epil.front_crc = -1; + con->v2.out_epil.middle_crc = -1; + con->v2.out_state = OUT_S_QUEUE_DATA; + return; + } + + if (front_len(msg)) { + con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base, + front_len(msg)); + add_out_kvec(con, msg->front.iov_base, front_len(msg)); + } else { + /* middle (at least) is there, checked above */ + con->v2.out_epil.front_crc = -1; + } + + if (middle_len(msg)) { + con->v2.out_epil.middle_crc = + crc32c(-1, msg->middle->vec.iov_base, middle_len(msg)); + add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg)); + } else { + con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0; + } + + if (data_len(msg)) { + con->v2.out_state = OUT_S_QUEUE_DATA; + } else { + con->v2.out_epil.data_crc = 0; + prepare_epilogue_plain(con, false); + con->v2.out_state = OUT_S_FINISH_MESSAGE; + } +} + +/* + * Unfortunately the kernel crypto API doesn't support streaming + * (piecewise) operation for AEAD algorithms, so we can't get away + * with a fixed size buffer and a couple sgs. Instead, we have to + * allocate pages for the entire tail of the message (currently up + * to ~32M) and two sgs arrays (up to ~256K each)... + */ +static int prepare_message_secure(struct ceph_connection *con) +{ + void *zerop = page_address(ceph_zero_page); + struct sg_table enc_sgt = {}; + struct sg_table sgt = {}; + struct page **enc_pages; + int enc_page_cnt; + int tail_len; + int ret; + + ret = prepare_head_secure_small(con, con->v2.out_buf, + sizeof(struct ceph_msg_header2)); + if (ret) + return ret; + + tail_len = tail_onwire_len(con->out_msg, true); + if (!tail_len) { + /* + * Empty message: once the head is written, + * we are done -- there is no epilogue. + */ + con->v2.out_state = OUT_S_FINISH_MESSAGE; + return 0; + } + + encode_epilogue_secure(con, false); + ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop, + &con->v2.out_epil, false); + if (ret) + goto out; + + enc_page_cnt = calc_pages_for(0, tail_len); + enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO); + if (IS_ERR(enc_pages)) { + ret = PTR_ERR(enc_pages); + goto out; + } + + WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt); + con->v2.out_enc_pages = enc_pages; + con->v2.out_enc_page_cnt = enc_page_cnt; + con->v2.out_enc_resid = tail_len; + con->v2.out_enc_i = 0; + + ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt, + 0, tail_len, GFP_NOIO); + if (ret) + goto out; + + ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl, + tail_len - CEPH_GCM_TAG_LEN); + if (ret) + goto out; + + dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con, + con->out_msg, sgt.orig_nents, enc_page_cnt); + con->v2.out_state = OUT_S_QUEUE_ENC_PAGE; + +out: + sg_free_table(&sgt); + sg_free_table(&enc_sgt); + return ret; +} + +static int prepare_message(struct ceph_connection *con) +{ + int lens[] = { + sizeof(struct ceph_msg_header2), + front_len(con->out_msg), + middle_len(con->out_msg), + data_len(con->out_msg) + }; + struct ceph_frame_desc desc; + int ret; + + dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con, + con->out_msg, lens[0], lens[1], lens[2], lens[3]); + + if (con->in_seq > con->in_seq_acked) { + dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con, + con->in_seq_acked, con->in_seq); + con->in_seq_acked = con->in_seq; + } + + reset_out_kvecs(con); + init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4); + encode_preamble(&desc, con->v2.out_buf); + fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr, + con->in_seq_acked); + + if (con_secure(con)) { + ret = prepare_message_secure(con); + if (ret) + return ret; + } else { + prepare_message_plain(con); + } + + ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); + return 0; +} + +static int prepare_read_banner_prefix(struct ceph_connection *con) +{ + void *buf; + + buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN); + if (!buf) + return -ENOMEM; + + reset_in_kvecs(con); + add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN); + add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN); + con->state = CEPH_CON_S_V2_BANNER_PREFIX; + return 0; +} + +static int prepare_read_banner_payload(struct ceph_connection *con, + int payload_len) +{ + void *buf; + + buf = alloc_conn_buf(con, payload_len); + if (!buf) + return -ENOMEM; + + reset_in_kvecs(con); + add_in_kvec(con, buf, payload_len); + add_in_sign_kvec(con, buf, payload_len); + con->state = CEPH_CON_S_V2_BANNER_PAYLOAD; + return 0; +} + +static void prepare_read_preamble(struct ceph_connection *con) +{ + reset_in_kvecs(con); + add_in_kvec(con, con->v2.in_buf, + con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN : + CEPH_PREAMBLE_PLAIN_LEN); + con->v2.in_state = IN_S_HANDLE_PREAMBLE; +} + +static int prepare_read_control(struct ceph_connection *con) +{ + int ctrl_len = con->v2.in_desc.fd_lens[0]; + int head_len; + void *buf; + + reset_in_kvecs(con); + if (con->state == CEPH_CON_S_V2_HELLO || + con->state == CEPH_CON_S_V2_AUTH) { + head_len = head_onwire_len(ctrl_len, false); + buf = alloc_conn_buf(con, head_len); + if (!buf) + return -ENOMEM; + + /* preserve preamble */ + memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN); + + add_in_kvec(con, CTRL_BODY(buf), ctrl_len); + add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN); + add_in_sign_kvec(con, buf, head_len); + } else { + if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) { + buf = alloc_conn_buf(con, ctrl_len); + if (!buf) + return -ENOMEM; + + add_in_kvec(con, buf, ctrl_len); + } else { + add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len); + } + add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN); + } + con->v2.in_state = IN_S_HANDLE_CONTROL; + return 0; +} + +static int prepare_read_control_remainder(struct ceph_connection *con) +{ + int ctrl_len = con->v2.in_desc.fd_lens[0]; + int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; + void *buf; + + buf = alloc_conn_buf(con, ctrl_len); + if (!buf) + return -ENOMEM; + + memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN); + + reset_in_kvecs(con); + add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len); + add_in_kvec(con, con->v2.in_buf, + padding_len(rem_len) + CEPH_GCM_TAG_LEN); + con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER; + return 0; +} + +static void prepare_read_data(struct ceph_connection *con) +{ + struct bio_vec bv; + + if (!con_secure(con)) + con->in_data_crc = -1; + ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg, + data_len(con->in_msg)); + + get_bvec_at(&con->v2.in_cursor, &bv); + set_in_bvec(con, &bv); + con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT; +} + +static void prepare_read_data_cont(struct ceph_connection *con) +{ + struct bio_vec bv; + + if (!con_secure(con)) + con->in_data_crc = ceph_crc32c_page(con->in_data_crc, + con->v2.in_bvec.bv_page, + con->v2.in_bvec.bv_offset, + con->v2.in_bvec.bv_len); + + ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len); + if (con->v2.in_cursor.total_resid) { + get_bvec_at(&con->v2.in_cursor, &bv); + set_in_bvec(con, &bv); + WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT); + return; + } + + /* + * We've read all data. Prepare to read data padding (if any) + * and epilogue. + */ + reset_in_kvecs(con); + if (con_secure(con)) { + if (need_padding(data_len(con->in_msg))) + add_in_kvec(con, DATA_PAD(con->v2.in_buf), + padding_len(data_len(con->in_msg))); + add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_SECURE_LEN); + } else { + add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); + } + con->v2.in_state = IN_S_HANDLE_EPILOGUE; +} + +static void __finish_skip(struct ceph_connection *con) +{ + con->in_seq++; + prepare_read_preamble(con); +} + +static void prepare_skip_message(struct ceph_connection *con) +{ + struct ceph_frame_desc *desc = &con->v2.in_desc; + int tail_len; + + dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1], + desc->fd_lens[2], desc->fd_lens[3]); + + tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2], + desc->fd_lens[3], con_secure(con)); + if (!tail_len) { + __finish_skip(con); + } else { + set_in_skip(con, tail_len); + con->v2.in_state = IN_S_FINISH_SKIP; + } +} + +static int process_banner_prefix(struct ceph_connection *con) +{ + int payload_len; + void *p; + + WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN); + + p = con->v2.in_kvecs[0].iov_base; + if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) { + if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN)) + con->error_msg = "server is speaking msgr1 protocol"; + else + con->error_msg = "protocol error, bad banner"; + return -EINVAL; + } + + p += CEPH_BANNER_V2_LEN; + payload_len = ceph_decode_16(&p); + dout("%s con %p payload_len %d\n", __func__, con, payload_len); + + return prepare_read_banner_payload(con, payload_len); +} + +static int process_banner_payload(struct ceph_connection *con) +{ + void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len; + u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES; + u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES; + u64 server_feat, server_req_feat; + void *p; + int ret; + + p = con->v2.in_kvecs[0].iov_base; + ceph_decode_64_safe(&p, end, server_feat, bad); + ceph_decode_64_safe(&p, end, server_req_feat, bad); + + dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n", + __func__, con, server_feat, server_req_feat); + + if (req_feat & ~server_feat) { + pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n", + server_feat, req_feat & ~server_feat); + con->error_msg = "missing required protocol features"; + return -EINVAL; + } + if (server_req_feat & ~feat) { + pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n", + feat, server_req_feat & ~feat); + con->error_msg = "missing required protocol features"; + return -EINVAL; + } + + /* no reset_out_kvecs() as our banner may still be pending */ + ret = prepare_hello(con); + if (ret) { + pr_err("prepare_hello failed: %d\n", ret); + return ret; + } + + con->state = CEPH_CON_S_V2_HELLO; + prepare_read_preamble(con); + return 0; + +bad: + pr_err("failed to decode banner payload\n"); + return -EINVAL; +} + +static int process_hello(struct ceph_connection *con, void *p, void *end) +{ + struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; + struct ceph_entity_addr addr_for_me; + u8 entity_type; + int ret; + + if (con->state != CEPH_CON_S_V2_HELLO) { + con->error_msg = "protocol error, unexpected hello"; + return -EINVAL; + } + + ceph_decode_8_safe(&p, end, entity_type, bad); + ret = ceph_decode_entity_addr(&p, end, &addr_for_me); + if (ret) { + pr_err("failed to decode addr_for_me: %d\n", ret); + return ret; + } + + dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con, + entity_type, ceph_pr_addr(&addr_for_me)); + + if (entity_type != con->peer_name.type) { + pr_err("bad peer type, want %d, got %d\n", + con->peer_name.type, entity_type); + con->error_msg = "wrong peer at address"; + return -EINVAL; + } + + /* + * Set our address to the address our first peer (i.e. monitor) + * sees that we are connecting from. If we are behind some sort + * of NAT and want to be identified by some private (not NATed) + * address, ip option should be used. + */ + if (ceph_addr_is_blank(my_addr)) { + memcpy(&my_addr->in_addr, &addr_for_me.in_addr, + sizeof(my_addr->in_addr)); + ceph_addr_set_port(my_addr, 0); + dout("%s con %p set my addr %s, as seen by peer %s\n", + __func__, con, ceph_pr_addr(my_addr), + ceph_pr_addr(&con->peer_addr)); + } else { + dout("%s con %p my addr already set %s\n", + __func__, con, ceph_pr_addr(my_addr)); + } + + WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr)); + WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY); + WARN_ON(!my_addr->nonce); + + /* no reset_out_kvecs() as our hello may still be pending */ + ret = prepare_auth_request(con); + if (ret) { + if (ret != -EAGAIN) + pr_err("prepare_auth_request failed: %d\n", ret); + return ret; + } + + con->state = CEPH_CON_S_V2_AUTH; + return 0; + +bad: + pr_err("failed to decode hello\n"); + return -EINVAL; +} + +static int process_auth_bad_method(struct ceph_connection *con, + void *p, void *end) +{ + int allowed_protos[8], allowed_modes[8]; + int allowed_proto_cnt, allowed_mode_cnt; + int used_proto, result; + int ret; + int i; + + if (con->state != CEPH_CON_S_V2_AUTH) { + con->error_msg = "protocol error, unexpected auth_bad_method"; + return -EINVAL; + } + + ceph_decode_32_safe(&p, end, used_proto, bad); + ceph_decode_32_safe(&p, end, result, bad); + dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto, + result); + + ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad); + if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) { + pr_err("allowed_protos too big %d\n", allowed_proto_cnt); + return -EINVAL; + } + for (i = 0; i < allowed_proto_cnt; i++) { + ceph_decode_32_safe(&p, end, allowed_protos[i], bad); + dout("%s con %p allowed_protos[%d] %d\n", __func__, con, + i, allowed_protos[i]); + } + + ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad); + if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) { + pr_err("allowed_modes too big %d\n", allowed_mode_cnt); + return -EINVAL; + } + for (i = 0; i < allowed_mode_cnt; i++) { + ceph_decode_32_safe(&p, end, allowed_modes[i], bad); + dout("%s con %p allowed_modes[%d] %d\n", __func__, con, + i, allowed_modes[i]); + } + + mutex_unlock(&con->mutex); + ret = con->ops->handle_auth_bad_method(con, used_proto, result, + allowed_protos, + allowed_proto_cnt, + allowed_modes, + allowed_mode_cnt); + mutex_lock(&con->mutex); + if (con->state != CEPH_CON_S_V2_AUTH) { + dout("%s con %p state changed to %d\n", __func__, con, + con->state); + return -EAGAIN; + } + + dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret); + return ret; + +bad: + pr_err("failed to decode auth_bad_method\n"); + return -EINVAL; +} + +static int process_auth_reply_more(struct ceph_connection *con, + void *p, void *end) +{ + int payload_len; + int ret; + + if (con->state != CEPH_CON_S_V2_AUTH) { + con->error_msg = "protocol error, unexpected auth_reply_more"; + return -EINVAL; + } + + ceph_decode_32_safe(&p, end, payload_len, bad); + ceph_decode_need(&p, end, payload_len, bad); + + dout("%s con %p payload_len %d\n", __func__, con, payload_len); + + reset_out_kvecs(con); + ret = prepare_auth_request_more(con, p, payload_len); + if (ret) { + if (ret != -EAGAIN) + pr_err("prepare_auth_request_more failed: %d\n", ret); + return ret; + } + + return 0; + +bad: + pr_err("failed to decode auth_reply_more\n"); + return -EINVAL; +} + +static int process_auth_done(struct ceph_connection *con, void *p, void *end) +{ + u8 session_key[CEPH_KEY_LEN]; + u8 con_secret[CEPH_MAX_CON_SECRET_LEN]; + int session_key_len, con_secret_len; + int payload_len; + u64 global_id; + int ret; + + if (con->state != CEPH_CON_S_V2_AUTH) { + con->error_msg = "protocol error, unexpected auth_done"; + return -EINVAL; + } + + ceph_decode_64_safe(&p, end, global_id, bad); + ceph_decode_32_safe(&p, end, con->v2.con_mode, bad); + ceph_decode_32_safe(&p, end, payload_len, bad); + + dout("%s con %p global_id %llu con_mode %d payload_len %d\n", + __func__, con, global_id, con->v2.con_mode, payload_len); + + mutex_unlock(&con->mutex); + session_key_len = 0; + con_secret_len = 0; + ret = con->ops->handle_auth_done(con, global_id, p, payload_len, + session_key, &session_key_len, + con_secret, &con_secret_len); + mutex_lock(&con->mutex); + if (con->state != CEPH_CON_S_V2_AUTH) { + dout("%s con %p state changed to %d\n", __func__, con, + con->state); + return -EAGAIN; + } + + dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret); + if (ret) + return ret; + + ret = setup_crypto(con, session_key, session_key_len, con_secret, + con_secret_len); + if (ret) + return ret; + + reset_out_kvecs(con); + ret = prepare_auth_signature(con); + if (ret) { + pr_err("prepare_auth_signature failed: %d\n", ret); + return ret; + } + + con->state = CEPH_CON_S_V2_AUTH_SIGNATURE; + return 0; + +bad: + pr_err("failed to decode auth_done\n"); + return -EINVAL; +} + +static int process_auth_signature(struct ceph_connection *con, + void *p, void *end) +{ + u8 hmac[SHA256_DIGEST_SIZE]; + int ret; + + if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) { + con->error_msg = "protocol error, unexpected auth_signature"; + return -EINVAL; + } + + ret = hmac_sha256(con, con->v2.out_sign_kvecs, + con->v2.out_sign_kvec_cnt, hmac); + if (ret) + return ret; + + ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad); + if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) { + con->error_msg = "integrity error, bad auth signature"; + return -EBADMSG; + } + + dout("%s con %p auth signature ok\n", __func__, con); + + /* no reset_out_kvecs() as our auth_signature may still be pending */ + if (!con->v2.server_cookie) { + ret = prepare_client_ident(con); + if (ret) { + pr_err("prepare_client_ident failed: %d\n", ret); + return ret; + } + + con->state = CEPH_CON_S_V2_SESSION_CONNECT; + } else { + ret = prepare_session_reconnect(con); + if (ret) { + pr_err("prepare_session_reconnect failed: %d\n", ret); + return ret; + } + + con->state = CEPH_CON_S_V2_SESSION_RECONNECT; + } + + return 0; + +bad: + pr_err("failed to decode auth_signature\n"); + return -EINVAL; +} + +static int process_server_ident(struct ceph_connection *con, + void *p, void *end) +{ + struct ceph_client *client = from_msgr(con->msgr); + u64 features, required_features; + struct ceph_entity_addr addr; + u64 global_seq; + u64 global_id; + u64 cookie; + u64 flags; + int ret; + + if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) { + con->error_msg = "protocol error, unexpected server_ident"; + return -EINVAL; + } + + ret = ceph_decode_entity_addrvec(&p, end, true, &addr); + if (ret) { + pr_err("failed to decode server addrs: %d\n", ret); + return ret; + } + + ceph_decode_64_safe(&p, end, global_id, bad); + ceph_decode_64_safe(&p, end, global_seq, bad); + ceph_decode_64_safe(&p, end, features, bad); + ceph_decode_64_safe(&p, end, required_features, bad); + ceph_decode_64_safe(&p, end, flags, bad); + ceph_decode_64_safe(&p, end, cookie, bad); + + dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n", + __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce), + global_id, global_seq, features, required_features, flags, cookie); + + /* is this who we intended to talk to? */ + if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) { + pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n", + ceph_pr_addr(&con->peer_addr), + le32_to_cpu(con->peer_addr.nonce), + ceph_pr_addr(&addr), le32_to_cpu(addr.nonce)); + con->error_msg = "wrong peer at address"; + return -EINVAL; + } + + if (client->required_features & ~features) { + pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n", + features, client->required_features & ~features); + con->error_msg = "missing required protocol features"; + return -EINVAL; + } + + /* + * Both name->type and name->num are set in ceph_con_open() but + * name->num may be bogus in the initial monmap. name->type is + * verified in handle_hello(). + */ + WARN_ON(!con->peer_name.type); + con->peer_name.num = cpu_to_le64(global_id); + con->v2.peer_global_seq = global_seq; + con->peer_features = features; + WARN_ON(required_features & ~client->supported_features); + con->v2.server_cookie = cookie; + + if (flags & CEPH_MSG_CONNECT_LOSSY) { + ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX); + WARN_ON(con->v2.server_cookie); + } else { + WARN_ON(!con->v2.server_cookie); + } + + clear_in_sign_kvecs(con); + clear_out_sign_kvecs(con); + free_conn_bufs(con); + con->delay = 0; /* reset backoff memory */ + + con->state = CEPH_CON_S_OPEN; + con->v2.out_state = OUT_S_GET_NEXT; + return 0; + +bad: + pr_err("failed to decode server_ident\n"); + return -EINVAL; +} + +static int process_ident_missing_features(struct ceph_connection *con, + void *p, void *end) +{ + struct ceph_client *client = from_msgr(con->msgr); + u64 missing_features; + + if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) { + con->error_msg = "protocol error, unexpected ident_missing_features"; + return -EINVAL; + } + + ceph_decode_64_safe(&p, end, missing_features, bad); + pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n", + client->supported_features, missing_features); + con->error_msg = "missing required protocol features"; + return -EINVAL; + +bad: + pr_err("failed to decode ident_missing_features\n"); + return -EINVAL; +} + +static int process_session_reconnect_ok(struct ceph_connection *con, + void *p, void *end) +{ + u64 seq; + + if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { + con->error_msg = "protocol error, unexpected session_reconnect_ok"; + return -EINVAL; + } + + ceph_decode_64_safe(&p, end, seq, bad); + + dout("%s con %p seq %llu\n", __func__, con, seq); + ceph_con_discard_requeued(con, seq); + + clear_in_sign_kvecs(con); + clear_out_sign_kvecs(con); + free_conn_bufs(con); + con->delay = 0; /* reset backoff memory */ + + con->state = CEPH_CON_S_OPEN; + con->v2.out_state = OUT_S_GET_NEXT; + return 0; + +bad: + pr_err("failed to decode session_reconnect_ok\n"); + return -EINVAL; +} + +static int process_session_retry(struct ceph_connection *con, + void *p, void *end) +{ + u64 connect_seq; + int ret; + + if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { + con->error_msg = "protocol error, unexpected session_retry"; + return -EINVAL; + } + + ceph_decode_64_safe(&p, end, connect_seq, bad); + + dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq); + WARN_ON(connect_seq <= con->v2.connect_seq); + con->v2.connect_seq = connect_seq + 1; + + free_conn_bufs(con); + + reset_out_kvecs(con); + ret = prepare_session_reconnect(con); + if (ret) { + pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret); + return ret; + } + + return 0; + +bad: + pr_err("failed to decode session_retry\n"); + return -EINVAL; +} + +static int process_session_retry_global(struct ceph_connection *con, + void *p, void *end) +{ + u64 global_seq; + int ret; + + if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { + con->error_msg = "protocol error, unexpected session_retry_global"; + return -EINVAL; + } + + ceph_decode_64_safe(&p, end, global_seq, bad); + + dout("%s con %p global_seq %llu\n", __func__, con, global_seq); + WARN_ON(global_seq <= con->v2.global_seq); + con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq); + + free_conn_bufs(con); + + reset_out_kvecs(con); + ret = prepare_session_reconnect(con); + if (ret) { + pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret); + return ret; + } + + return 0; + +bad: + pr_err("failed to decode session_retry_global\n"); + return -EINVAL; +} + +static int process_session_reset(struct ceph_connection *con, + void *p, void *end) +{ + bool full; + int ret; + + if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { + con->error_msg = "protocol error, unexpected session_reset"; + return -EINVAL; + } + + ceph_decode_8_safe(&p, end, full, bad); + if (!full) { + con->error_msg = "protocol error, bad session_reset"; + return -EINVAL; + } + + pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name), + ceph_pr_addr(&con->peer_addr)); + ceph_con_reset_session(con); + + mutex_unlock(&con->mutex); + if (con->ops->peer_reset) + con->ops->peer_reset(con); + mutex_lock(&con->mutex); + if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { + dout("%s con %p state changed to %d\n", __func__, con, + con->state); + return -EAGAIN; + } + + free_conn_bufs(con); + + reset_out_kvecs(con); + ret = prepare_client_ident(con); + if (ret) { + pr_err("prepare_client_ident (rst) failed: %d\n", ret); + return ret; + } + + con->state = CEPH_CON_S_V2_SESSION_CONNECT; + return 0; + +bad: + pr_err("failed to decode session_reset\n"); + return -EINVAL; +} + +static int process_keepalive2_ack(struct ceph_connection *con, + void *p, void *end) +{ + if (con->state != CEPH_CON_S_OPEN) { + con->error_msg = "protocol error, unexpected keepalive2_ack"; + return -EINVAL; + } + + ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad); + ceph_decode_timespec64(&con->last_keepalive_ack, p); + + dout("%s con %p timestamp %lld.%09ld\n", __func__, con, + con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec); + + return 0; + +bad: + pr_err("failed to decode keepalive2_ack\n"); + return -EINVAL; +} + +static int process_ack(struct ceph_connection *con, void *p, void *end) +{ + u64 seq; + + if (con->state != CEPH_CON_S_OPEN) { + con->error_msg = "protocol error, unexpected ack"; + return -EINVAL; + } + + ceph_decode_64_safe(&p, end, seq, bad); + + dout("%s con %p seq %llu\n", __func__, con, seq); + ceph_con_discard_sent(con, seq); + return 0; + +bad: + pr_err("failed to decode ack\n"); + return -EINVAL; +} + +static int process_control(struct ceph_connection *con, void *p, void *end) +{ + int tag = con->v2.in_desc.fd_tag; + int ret; + + dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p)); + + switch (tag) { + case FRAME_TAG_HELLO: + ret = process_hello(con, p, end); + break; + case FRAME_TAG_AUTH_BAD_METHOD: + ret = process_auth_bad_method(con, p, end); + break; + case FRAME_TAG_AUTH_REPLY_MORE: + ret = process_auth_reply_more(con, p, end); + break; + case FRAME_TAG_AUTH_DONE: + ret = process_auth_done(con, p, end); + break; + case FRAME_TAG_AUTH_SIGNATURE: + ret = process_auth_signature(con, p, end); + break; + case FRAME_TAG_SERVER_IDENT: + ret = process_server_ident(con, p, end); + break; + case FRAME_TAG_IDENT_MISSING_FEATURES: + ret = process_ident_missing_features(con, p, end); + break; + case FRAME_TAG_SESSION_RECONNECT_OK: + ret = process_session_reconnect_ok(con, p, end); + break; + case FRAME_TAG_SESSION_RETRY: + ret = process_session_retry(con, p, end); + break; + case FRAME_TAG_SESSION_RETRY_GLOBAL: + ret = process_session_retry_global(con, p, end); + break; + case FRAME_TAG_SESSION_RESET: + ret = process_session_reset(con, p, end); + break; + case FRAME_TAG_KEEPALIVE2_ACK: + ret = process_keepalive2_ack(con, p, end); + break; + case FRAME_TAG_ACK: + ret = process_ack(con, p, end); + break; + default: + pr_err("bad tag %d\n", tag); + con->error_msg = "protocol error, bad tag"; + return -EINVAL; + } + if (ret) { + dout("%s con %p error %d\n", __func__, con, ret); + return ret; + } + + prepare_read_preamble(con); + return 0; +} + +/* + * Return: + * 1 - con->in_msg set, read message + * 0 - skip message + * <0 - error + */ +static int process_message_header(struct ceph_connection *con, + void *p, void *end) +{ + struct ceph_frame_desc *desc = &con->v2.in_desc; + struct ceph_msg_header2 *hdr2 = p; + struct ceph_msg_header hdr; + int skip; + int ret; + u64 seq; + + /* verify seq# */ + seq = le64_to_cpu(hdr2->seq); + if ((s64)seq - (s64)con->in_seq < 1) { + pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n", + ENTITY_NAME(con->peer_name), + ceph_pr_addr(&con->peer_addr), + seq, con->in_seq + 1); + return 0; + } + if ((s64)seq - (s64)con->in_seq > 1) { + pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1); + con->error_msg = "bad message sequence # for incoming message"; + return -EBADE; + } + + ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq)); + + fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2], + desc->fd_lens[3], &con->peer_name); + ret = ceph_con_in_msg_alloc(con, &hdr, &skip); + if (ret) + return ret; + + WARN_ON(!con->in_msg ^ skip); + if (skip) + return 0; + + WARN_ON(!con->in_msg); + WARN_ON(con->in_msg->con != con); + return 1; +} + +static int process_message(struct ceph_connection *con) +{ + ceph_con_process_message(con); + + /* + * We could have been closed by ceph_con_close() because + * ceph_con_process_message() temporarily drops con->mutex. + */ + if (con->state != CEPH_CON_S_OPEN) { + dout("%s con %p state changed to %d\n", __func__, con, + con->state); + return -EAGAIN; + } + + prepare_read_preamble(con); + return 0; +} + +static int __handle_control(struct ceph_connection *con, void *p) +{ + void *end = p + con->v2.in_desc.fd_lens[0]; + struct ceph_msg *msg; + int ret; + + if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE) + return process_control(con, p, end); + + ret = process_message_header(con, p, end); + if (ret < 0) + return ret; + if (ret == 0) { + prepare_skip_message(con); + return 0; + } + + msg = con->in_msg; /* set in process_message_header() */ + if (!front_len(msg) && !middle_len(msg)) { + if (!data_len(msg)) + return process_message(con); + + prepare_read_data(con); + return 0; + } + + reset_in_kvecs(con); + if (front_len(msg)) { + WARN_ON(front_len(msg) > msg->front_alloc_len); + add_in_kvec(con, msg->front.iov_base, front_len(msg)); + msg->front.iov_len = front_len(msg); + + if (con_secure(con) && need_padding(front_len(msg))) + add_in_kvec(con, FRONT_PAD(con->v2.in_buf), + padding_len(front_len(msg))); + } else { + msg->front.iov_len = 0; + } + if (middle_len(msg)) { + WARN_ON(middle_len(msg) > msg->middle->alloc_len); + add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg)); + msg->middle->vec.iov_len = middle_len(msg); + + if (con_secure(con) && need_padding(middle_len(msg))) + add_in_kvec(con, MIDDLE_PAD(con->v2.in_buf), + padding_len(middle_len(msg))); + } else if (msg->middle) { + msg->middle->vec.iov_len = 0; + } + + if (data_len(msg)) { + con->v2.in_state = IN_S_PREPARE_READ_DATA; + } else { + add_in_kvec(con, con->v2.in_buf, + con_secure(con) ? CEPH_EPILOGUE_SECURE_LEN : + CEPH_EPILOGUE_PLAIN_LEN); + con->v2.in_state = IN_S_HANDLE_EPILOGUE; + } + return 0; +} + +static int handle_preamble(struct ceph_connection *con) +{ + struct ceph_frame_desc *desc = &con->v2.in_desc; + int ret; + + if (con_secure(con)) { + ret = decrypt_preamble(con); + if (ret) { + if (ret == -EBADMSG) + con->error_msg = "integrity error, bad preamble auth tag"; + return ret; + } + } + + ret = decode_preamble(con->v2.in_buf, desc); + if (ret) { + if (ret == -EBADMSG) + con->error_msg = "integrity error, bad crc"; + else + con->error_msg = "protocol error, bad preamble"; + return ret; + } + + dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__, + con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0], + desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]); + + if (!con_secure(con)) + return prepare_read_control(con); + + if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN) + return prepare_read_control_remainder(con); + + return __handle_control(con, CTRL_BODY(con->v2.in_buf)); +} + +static int handle_control(struct ceph_connection *con) +{ + int ctrl_len = con->v2.in_desc.fd_lens[0]; + void *buf; + int ret; + + WARN_ON(con_secure(con)); + + ret = verify_control_crc(con); + if (ret) { + con->error_msg = "integrity error, bad crc"; + return ret; + } + + if (con->state == CEPH_CON_S_V2_AUTH) { + buf = alloc_conn_buf(con, ctrl_len); + if (!buf) + return -ENOMEM; + + memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len); + return __handle_control(con, buf); + } + + return __handle_control(con, con->v2.in_kvecs[0].iov_base); +} + +static int handle_control_remainder(struct ceph_connection *con) +{ + int ret; + + WARN_ON(!con_secure(con)); + + ret = decrypt_control_remainder(con); + if (ret) { + if (ret == -EBADMSG) + con->error_msg = "integrity error, bad control remainder auth tag"; + return ret; + } + + return __handle_control(con, con->v2.in_kvecs[0].iov_base - + CEPH_PREAMBLE_INLINE_LEN); +} + +static int handle_epilogue(struct ceph_connection *con) +{ + u32 front_crc, middle_crc, data_crc; + int ret; + + if (con_secure(con)) { + ret = decrypt_message(con); + if (ret) { + if (ret == -EBADMSG) + con->error_msg = "integrity error, bad epilogue auth tag"; + return ret; + } + + /* just late_status */ + ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL); + if (ret) { + con->error_msg = "protocol error, bad epilogue"; + return ret; + } + } else { + ret = decode_epilogue(con->v2.in_buf, &front_crc, + &middle_crc, &data_crc); + if (ret) { + con->error_msg = "protocol error, bad epilogue"; + return ret; + } + + ret = verify_epilogue_crcs(con, front_crc, middle_crc, + data_crc); + if (ret) { + con->error_msg = "integrity error, bad crc"; + return ret; + } + } + + return process_message(con); +} + +static void finish_skip(struct ceph_connection *con) +{ + dout("%s con %p\n", __func__, con); + + if (con_secure(con)) + gcm_inc_nonce(&con->v2.in_gcm_nonce); + + __finish_skip(con); +} + +static int populate_in_iter(struct ceph_connection *con) +{ + int ret; + + dout("%s con %p state %d in_state %d\n", __func__, con, con->state, + con->v2.in_state); + WARN_ON(iov_iter_count(&con->v2.in_iter)); + + if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) { + ret = process_banner_prefix(con); + } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) { + ret = process_banner_payload(con); + } else if ((con->state >= CEPH_CON_S_V2_HELLO && + con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) || + con->state == CEPH_CON_S_OPEN) { + switch (con->v2.in_state) { + case IN_S_HANDLE_PREAMBLE: + ret = handle_preamble(con); + break; + case IN_S_HANDLE_CONTROL: + ret = handle_control(con); + break; + case IN_S_HANDLE_CONTROL_REMAINDER: + ret = handle_control_remainder(con); + break; + case IN_S_PREPARE_READ_DATA: + prepare_read_data(con); + ret = 0; + break; + case IN_S_PREPARE_READ_DATA_CONT: + prepare_read_data_cont(con); + ret = 0; + break; + case IN_S_HANDLE_EPILOGUE: + ret = handle_epilogue(con); + break; + case IN_S_FINISH_SKIP: + finish_skip(con); + ret = 0; + break; + default: + WARN(1, "bad in_state %d", con->v2.in_state); + return -EINVAL; + } + } else { + WARN(1, "bad state %d", con->state); + return -EINVAL; + } + if (ret) { + dout("%s con %p error %d\n", __func__, con, ret); + return ret; + } + + if (WARN_ON(!iov_iter_count(&con->v2.in_iter))) + return -ENODATA; + dout("%s con %p populated %zu\n", __func__, con, + iov_iter_count(&con->v2.in_iter)); + return 1; +} + +int ceph_con_v2_try_read(struct ceph_connection *con) +{ + int ret; + + dout("%s con %p state %d need %zu\n", __func__, con, con->state, + iov_iter_count(&con->v2.in_iter)); + + if (con->state == CEPH_CON_S_PREOPEN) + return 0; + + /* + * We should always have something pending here. If not, + * avoid calling populate_in_iter() as if we read something + * (ceph_tcp_recv() would immediately return 1). + */ + if (WARN_ON(!iov_iter_count(&con->v2.in_iter))) + return -ENODATA; + + for (;;) { + ret = ceph_tcp_recv(con); + if (ret <= 0) + return ret; + + ret = populate_in_iter(con); + if (ret <= 0) { + if (ret && ret != -EAGAIN && !con->error_msg) + con->error_msg = "read processing error"; + return ret; + } + } +} + +static void queue_data(struct ceph_connection *con) +{ + struct bio_vec bv; + + con->v2.out_epil.data_crc = -1; + ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg, + data_len(con->out_msg)); + + get_bvec_at(&con->v2.out_cursor, &bv); + set_out_bvec(con, &bv, true); + con->v2.out_state = OUT_S_QUEUE_DATA_CONT; +} + +static void queue_data_cont(struct ceph_connection *con) +{ + struct bio_vec bv; + + con->v2.out_epil.data_crc = ceph_crc32c_page( + con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page, + con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len); + + ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len); + if (con->v2.out_cursor.total_resid) { + get_bvec_at(&con->v2.out_cursor, &bv); + set_out_bvec(con, &bv, true); + WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT); + return; + } + + /* + * We've written all data. Queue epilogue. Once it's written, + * we are done. + */ + reset_out_kvecs(con); + prepare_epilogue_plain(con, false); + con->v2.out_state = OUT_S_FINISH_MESSAGE; +} + +static void queue_enc_page(struct ceph_connection *con) +{ + struct bio_vec bv; + + dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i, + con->v2.out_enc_resid); + WARN_ON(!con->v2.out_enc_resid); + + bv.bv_page = con->v2.out_enc_pages[con->v2.out_enc_i]; + bv.bv_offset = 0; + bv.bv_len = min(con->v2.out_enc_resid, (int)PAGE_SIZE); + + set_out_bvec(con, &bv, false); + con->v2.out_enc_i++; + con->v2.out_enc_resid -= bv.bv_len; + + if (con->v2.out_enc_resid) { + WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE); + return; + } + + /* + * We've queued the last piece of ciphertext (ending with + * epilogue) + auth tag. Once it's written, we are done. + */ + WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt); + con->v2.out_state = OUT_S_FINISH_MESSAGE; +} + +static void queue_zeros(struct ceph_connection *con) +{ + dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero); + + if (con->v2.out_zero) { + set_out_bvec_zero(con); + con->v2.out_zero -= con->v2.out_bvec.bv_len; + con->v2.out_state = OUT_S_QUEUE_ZEROS; + return; + } + + /* + * We've zero-filled everything up to epilogue. Queue epilogue + * with late_status set to ABORTED and crcs adjusted for zeros. + * Once it's written, we are done patching up for the revoke. + */ + reset_out_kvecs(con); + prepare_epilogue_plain(con, true); + con->v2.out_state = OUT_S_FINISH_MESSAGE; +} + +static void finish_message(struct ceph_connection *con) +{ + dout("%s con %p msg %p\n", __func__, con, con->out_msg); + + /* we end up here both plain and secure modes */ + if (con->v2.out_enc_pages) { + WARN_ON(!con->v2.out_enc_page_cnt); + ceph_release_page_vector(con->v2.out_enc_pages, + con->v2.out_enc_page_cnt); + con->v2.out_enc_pages = NULL; + con->v2.out_enc_page_cnt = 0; + } + /* message may have been revoked */ + if (con->out_msg) { + ceph_msg_put(con->out_msg); + con->out_msg = NULL; + } + + con->v2.out_state = OUT_S_GET_NEXT; +} + +static int populate_out_iter(struct ceph_connection *con) +{ + int ret; + + dout("%s con %p state %d out_state %d\n", __func__, con, con->state, + con->v2.out_state); + WARN_ON(iov_iter_count(&con->v2.out_iter)); + + if (con->state != CEPH_CON_S_OPEN) { + WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX || + con->state > CEPH_CON_S_V2_SESSION_RECONNECT); + goto nothing_pending; + } + + switch (con->v2.out_state) { + case OUT_S_QUEUE_DATA: + WARN_ON(!con->out_msg); + queue_data(con); + goto populated; + case OUT_S_QUEUE_DATA_CONT: + WARN_ON(!con->out_msg); + queue_data_cont(con); + goto populated; + case OUT_S_QUEUE_ENC_PAGE: + queue_enc_page(con); + goto populated; + case OUT_S_QUEUE_ZEROS: + WARN_ON(con->out_msg); /* revoked */ + queue_zeros(con); + goto populated; + case OUT_S_FINISH_MESSAGE: + finish_message(con); + break; + case OUT_S_GET_NEXT: + break; + default: + WARN(1, "bad out_state %d", con->v2.out_state); + return -EINVAL; + } + + WARN_ON(con->v2.out_state != OUT_S_GET_NEXT); + if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) { + ret = prepare_keepalive2(con); + if (ret) { + pr_err("prepare_keepalive2 failed: %d\n", ret); + return ret; + } + } else if (!list_empty(&con->out_queue)) { + ceph_con_get_out_msg(con); + ret = prepare_message(con); + if (ret) { + pr_err("prepare_message failed: %d\n", ret); + return ret; + } + } else if (con->in_seq > con->in_seq_acked) { + ret = prepare_ack(con); + if (ret) { + pr_err("prepare_ack failed: %d\n", ret); + return ret; + } + } else { + goto nothing_pending; + } + +populated: + if (WARN_ON(!iov_iter_count(&con->v2.out_iter))) + return -ENODATA; + dout("%s con %p populated %zu\n", __func__, con, + iov_iter_count(&con->v2.out_iter)); + return 1; + +nothing_pending: + WARN_ON(iov_iter_count(&con->v2.out_iter)); + dout("%s con %p nothing pending\n", __func__, con); + ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING); + return 0; +} + +int ceph_con_v2_try_write(struct ceph_connection *con) +{ + int ret; + + dout("%s con %p state %d have %zu\n", __func__, con, con->state, + iov_iter_count(&con->v2.out_iter)); + + /* open the socket first? */ + if (con->state == CEPH_CON_S_PREOPEN) { + WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2); + + /* + * Always bump global_seq. Bump connect_seq only if + * there is a session (i.e. we are reconnecting and will + * send session_reconnect instead of client_ident). + */ + con->v2.global_seq = ceph_get_global_seq(con->msgr, 0); + if (con->v2.server_cookie) + con->v2.connect_seq++; + + ret = prepare_read_banner_prefix(con); + if (ret) { + pr_err("prepare_read_banner_prefix failed: %d\n", ret); + con->error_msg = "connect error"; + return ret; + } + + reset_out_kvecs(con); + ret = prepare_banner(con); + if (ret) { + pr_err("prepare_banner failed: %d\n", ret); + con->error_msg = "connect error"; + return ret; + } + + ret = ceph_tcp_connect(con); + if (ret) { + pr_err("ceph_tcp_connect failed: %d\n", ret); + con->error_msg = "connect error"; + return ret; + } + } + + if (!iov_iter_count(&con->v2.out_iter)) { + ret = populate_out_iter(con); + if (ret <= 0) { + if (ret && ret != -EAGAIN && !con->error_msg) + con->error_msg = "write processing error"; + return ret; + } + } + + tcp_sock_set_cork(con->sock->sk, true); + for (;;) { + ret = ceph_tcp_send(con); + if (ret <= 0) + break; + + ret = populate_out_iter(con); + if (ret <= 0) { + if (ret && ret != -EAGAIN && !con->error_msg) + con->error_msg = "write processing error"; + break; + } + } + + tcp_sock_set_cork(con->sock->sk, false); + return ret; +} + +static u32 crc32c_zeros(u32 crc, int zero_len) +{ + int len; + + while (zero_len) { + len = min(zero_len, (int)PAGE_SIZE); + crc = crc32c(crc, page_address(ceph_zero_page), len); + zero_len -= len; + } + + return crc; +} + +static void prepare_zero_front(struct ceph_connection *con, int resid) +{ + int sent; + + WARN_ON(!resid || resid > front_len(con->out_msg)); + sent = front_len(con->out_msg) - resid; + dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); + + if (sent) { + con->v2.out_epil.front_crc = + crc32c(-1, con->out_msg->front.iov_base, sent); + con->v2.out_epil.front_crc = + crc32c_zeros(con->v2.out_epil.front_crc, resid); + } else { + con->v2.out_epil.front_crc = crc32c_zeros(-1, resid); + } + + con->v2.out_iter.count -= resid; + out_zero_add(con, resid); +} + +static void prepare_zero_middle(struct ceph_connection *con, int resid) +{ + int sent; + + WARN_ON(!resid || resid > middle_len(con->out_msg)); + sent = middle_len(con->out_msg) - resid; + dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); + + if (sent) { + con->v2.out_epil.middle_crc = + crc32c(-1, con->out_msg->middle->vec.iov_base, sent); + con->v2.out_epil.middle_crc = + crc32c_zeros(con->v2.out_epil.middle_crc, resid); + } else { + con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid); + } + + con->v2.out_iter.count -= resid; + out_zero_add(con, resid); +} + +static void prepare_zero_data(struct ceph_connection *con) +{ + dout("%s con %p\n", __func__, con); + con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg)); + out_zero_add(con, data_len(con->out_msg)); +} + +static void revoke_at_queue_data(struct ceph_connection *con) +{ + int boundary; + int resid; + + WARN_ON(!data_len(con->out_msg)); + WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); + resid = iov_iter_count(&con->v2.out_iter); + + boundary = front_len(con->out_msg) + middle_len(con->out_msg); + if (resid > boundary) { + resid -= boundary; + WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN); + dout("%s con %p was sending head\n", __func__, con); + if (front_len(con->out_msg)) + prepare_zero_front(con, front_len(con->out_msg)); + if (middle_len(con->out_msg)) + prepare_zero_middle(con, middle_len(con->out_msg)); + prepare_zero_data(con); + WARN_ON(iov_iter_count(&con->v2.out_iter) != resid); + con->v2.out_state = OUT_S_QUEUE_ZEROS; + return; + } + + boundary = middle_len(con->out_msg); + if (resid > boundary) { + resid -= boundary; + dout("%s con %p was sending front\n", __func__, con); + prepare_zero_front(con, resid); + if (middle_len(con->out_msg)) + prepare_zero_middle(con, middle_len(con->out_msg)); + prepare_zero_data(con); + queue_zeros(con); + return; + } + + WARN_ON(!resid); + dout("%s con %p was sending middle\n", __func__, con); + prepare_zero_middle(con, resid); + prepare_zero_data(con); + queue_zeros(con); +} + +static void revoke_at_queue_data_cont(struct ceph_connection *con) +{ + int sent, resid; /* current piece of data */ + + WARN_ON(!data_len(con->out_msg)); + WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter)); + resid = iov_iter_count(&con->v2.out_iter); + WARN_ON(!resid || resid > con->v2.out_bvec.bv_len); + sent = con->v2.out_bvec.bv_len - resid; + dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); + + if (sent) { + con->v2.out_epil.data_crc = ceph_crc32c_page( + con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page, + con->v2.out_bvec.bv_offset, sent); + ceph_msg_data_advance(&con->v2.out_cursor, sent); + } + WARN_ON(resid > con->v2.out_cursor.total_resid); + con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc, + con->v2.out_cursor.total_resid); + + con->v2.out_iter.count -= resid; + out_zero_add(con, con->v2.out_cursor.total_resid); + queue_zeros(con); +} + +static void revoke_at_finish_message(struct ceph_connection *con) +{ + int boundary; + int resid; + + WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); + resid = iov_iter_count(&con->v2.out_iter); + + if (!front_len(con->out_msg) && !middle_len(con->out_msg) && + !data_len(con->out_msg)) { + WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN); + dout("%s con %p was sending head (empty message) - noop\n", + __func__, con); + return; + } + + boundary = front_len(con->out_msg) + middle_len(con->out_msg) + + CEPH_EPILOGUE_PLAIN_LEN; + if (resid > boundary) { + resid -= boundary; + WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN); + dout("%s con %p was sending head\n", __func__, con); + if (front_len(con->out_msg)) + prepare_zero_front(con, front_len(con->out_msg)); + if (middle_len(con->out_msg)) + prepare_zero_middle(con, middle_len(con->out_msg)); + con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; + WARN_ON(iov_iter_count(&con->v2.out_iter) != resid); + con->v2.out_state = OUT_S_QUEUE_ZEROS; + return; + } + + boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN; + if (resid > boundary) { + resid -= boundary; + dout("%s con %p was sending front\n", __func__, con); + prepare_zero_front(con, resid); + if (middle_len(con->out_msg)) + prepare_zero_middle(con, middle_len(con->out_msg)); + con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; + queue_zeros(con); + return; + } + + boundary = CEPH_EPILOGUE_PLAIN_LEN; + if (resid > boundary) { + resid -= boundary; + dout("%s con %p was sending middle\n", __func__, con); + prepare_zero_middle(con, resid); + con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; + queue_zeros(con); + return; + } + + WARN_ON(!resid); + dout("%s con %p was sending epilogue - noop\n", __func__, con); +} + +void ceph_con_v2_revoke(struct ceph_connection *con) +{ + WARN_ON(con->v2.out_zero); + + if (con_secure(con)) { + WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE && + con->v2.out_state != OUT_S_FINISH_MESSAGE); + dout("%s con %p secure - noop\n", __func__, con); + return; + } + + switch (con->v2.out_state) { + case OUT_S_QUEUE_DATA: + revoke_at_queue_data(con); + break; + case OUT_S_QUEUE_DATA_CONT: + revoke_at_queue_data_cont(con); + break; + case OUT_S_FINISH_MESSAGE: + revoke_at_finish_message(con); + break; + default: + WARN(1, "bad out_state %d", con->v2.out_state); + break; + } +} + +static void revoke_at_prepare_read_data(struct ceph_connection *con) +{ + int remaining; /* data + [data padding] + epilogue */ + int resid; + + WARN_ON(!data_len(con->in_msg)); + WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); + resid = iov_iter_count(&con->v2.in_iter); + WARN_ON(!resid); + + if (con_secure(con)) + remaining = padded_len(data_len(con->in_msg)) + + CEPH_EPILOGUE_SECURE_LEN; + else + remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN; + + dout("%s con %p resid %d remaining %d\n", __func__, con, resid, + remaining); + con->v2.in_iter.count -= resid; + set_in_skip(con, resid + remaining); + con->v2.in_state = IN_S_FINISH_SKIP; +} + +static void revoke_at_prepare_read_data_cont(struct ceph_connection *con) +{ + int recved, resid; /* current piece of data */ + int remaining; /* [data padding] + epilogue */ + + WARN_ON(!data_len(con->in_msg)); + WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); + resid = iov_iter_count(&con->v2.in_iter); + WARN_ON(!resid || resid > con->v2.in_bvec.bv_len); + recved = con->v2.in_bvec.bv_len - resid; + dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid); + + if (recved) + ceph_msg_data_advance(&con->v2.in_cursor, recved); + WARN_ON(resid > con->v2.in_cursor.total_resid); + + if (con_secure(con)) + remaining = padding_len(data_len(con->in_msg)) + + CEPH_EPILOGUE_SECURE_LEN; + else + remaining = CEPH_EPILOGUE_PLAIN_LEN; + + dout("%s con %p total_resid %zu remaining %d\n", __func__, con, + con->v2.in_cursor.total_resid, remaining); + con->v2.in_iter.count -= resid; + set_in_skip(con, con->v2.in_cursor.total_resid + remaining); + con->v2.in_state = IN_S_FINISH_SKIP; +} + +static void revoke_at_handle_epilogue(struct ceph_connection *con) +{ + int resid; + + WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); + resid = iov_iter_count(&con->v2.in_iter); + WARN_ON(!resid); + + dout("%s con %p resid %d\n", __func__, con, resid); + con->v2.in_iter.count -= resid; + set_in_skip(con, resid); + con->v2.in_state = IN_S_FINISH_SKIP; +} + +void ceph_con_v2_revoke_incoming(struct ceph_connection *con) +{ + switch (con->v2.in_state) { + case IN_S_PREPARE_READ_DATA: + revoke_at_prepare_read_data(con); + break; + case IN_S_PREPARE_READ_DATA_CONT: + revoke_at_prepare_read_data_cont(con); + break; + case IN_S_HANDLE_EPILOGUE: + revoke_at_handle_epilogue(con); + break; + default: + WARN(1, "bad in_state %d", con->v2.in_state); + break; + } +} + +bool ceph_con_v2_opened(struct ceph_connection *con) +{ + return con->v2.peer_global_seq; +} + +void ceph_con_v2_reset_session(struct ceph_connection *con) +{ + con->v2.client_cookie = 0; + con->v2.server_cookie = 0; + con->v2.global_seq = 0; + con->v2.connect_seq = 0; + con->v2.peer_global_seq = 0; +} + +void ceph_con_v2_reset_protocol(struct ceph_connection *con) +{ + iov_iter_truncate(&con->v2.in_iter, 0); + iov_iter_truncate(&con->v2.out_iter, 0); + con->v2.out_zero = 0; + + clear_in_sign_kvecs(con); + clear_out_sign_kvecs(con); + free_conn_bufs(con); + + if (con->v2.out_enc_pages) { + WARN_ON(!con->v2.out_enc_page_cnt); + ceph_release_page_vector(con->v2.out_enc_pages, + con->v2.out_enc_page_cnt); + con->v2.out_enc_pages = NULL; + con->v2.out_enc_page_cnt = 0; + } + + con->v2.con_mode = CEPH_CON_MODE_UNKNOWN; + + if (con->v2.hmac_tfm) { + crypto_free_shash(con->v2.hmac_tfm); + con->v2.hmac_tfm = NULL; + } + if (con->v2.gcm_req) { + aead_request_free(con->v2.gcm_req); + con->v2.gcm_req = NULL; + } + if (con->v2.gcm_tfm) { + crypto_free_aead(con->v2.gcm_tfm); + con->v2.gcm_tfm = NULL; + } +} diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 792a8c4164d7..b9d54ed9f338 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -257,10 +257,16 @@ static void __open_session(struct ceph_mon_client *monc) &monc->monmap->mon_inst[monc->cur_mon].addr); /* - * send an initial keepalive to ensure our timestamp is valid - * by the time we are in an OPENED state + * Queue a keepalive to ensure that in case of an early fault + * the messenger doesn't put us into STANDBY state and instead + * retries. This also ensures that our timestamp is valid by + * the time we finish hunting and delayed_work() checks it. */ ceph_con_keepalive(&monc->con); + if (ceph_msgr2(monc->client)) { + monc->pending_auth = 1; + return; + } /* initiate authentication handshake */ ret = ceph_auth_build_hello(monc->auth, @@ -543,7 +549,7 @@ static void ceph_monc_handle_map(struct ceph_mon_client *monc, p = msg->front.iov_base; end = p + msg->front.iov_len; - monmap = ceph_monmap_decode(&p, end, false); + monmap = ceph_monmap_decode(&p, end, ceph_msgr2(client)); if (IS_ERR(monmap)) { pr_err("problem decoding monmap, %d\n", (int)PTR_ERR(monmap)); @@ -1119,8 +1125,9 @@ static void delayed_work(struct work_struct *work) */ static int build_initial_monmap(struct ceph_mon_client *monc) { + __le32 my_type = ceph_msgr2(monc->client) ? + CEPH_ENTITY_ADDR_TYPE_MSGR2 : CEPH_ENTITY_ADDR_TYPE_LEGACY; struct ceph_options *opt = monc->client->options; - struct ceph_entity_addr *mon_addr = opt->mon_addr; int num_mon = opt->num_mon; int i; @@ -1129,12 +1136,16 @@ static int build_initial_monmap(struct ceph_mon_client *monc) GFP_KERNEL); if (!monc->monmap) return -ENOMEM; + for (i = 0; i < num_mon; i++) { - monc->monmap->mon_inst[i].addr = mon_addr[i]; - monc->monmap->mon_inst[i].addr.nonce = 0; - monc->monmap->mon_inst[i].name.type = - CEPH_ENTITY_TYPE_MON; - monc->monmap->mon_inst[i].name.num = cpu_to_le64(i); + struct ceph_entity_inst *inst = &monc->monmap->mon_inst[i]; + + memcpy(&inst->addr.in_addr, &opt->mon_addr[i].in_addr, + sizeof(inst->addr.in_addr)); + inst->addr.type = my_type; + inst->addr.nonce = 0; + inst->name.type = CEPH_ENTITY_TYPE_MON; + inst->name.num = cpu_to_le64(i); } monc->monmap->num_mon = num_mon; return 0; @@ -1337,6 +1348,88 @@ int ceph_monc_validate_auth(struct ceph_mon_client *monc) } EXPORT_SYMBOL(ceph_monc_validate_auth); +static int mon_get_auth_request(struct ceph_connection *con, + void *buf, int *buf_len, + void **authorizer, int *authorizer_len) +{ + struct ceph_mon_client *monc = con->private; + int ret; + + mutex_lock(&monc->mutex); + ret = ceph_auth_get_request(monc->auth, buf, *buf_len); + mutex_unlock(&monc->mutex); + if (ret < 0) + return ret; + + *buf_len = ret; + *authorizer = NULL; + *authorizer_len = 0; + return 0; +} + +static int mon_handle_auth_reply_more(struct ceph_connection *con, + void *reply, int reply_len, + void *buf, int *buf_len, + void **authorizer, int *authorizer_len) +{ + struct ceph_mon_client *monc = con->private; + int ret; + + mutex_lock(&monc->mutex); + ret = ceph_auth_handle_reply_more(monc->auth, reply, reply_len, + buf, *buf_len); + mutex_unlock(&monc->mutex); + if (ret < 0) + return ret; + + *buf_len = ret; + *authorizer = NULL; + *authorizer_len = 0; + return 0; +} + +static int mon_handle_auth_done(struct ceph_connection *con, + u64 global_id, void *reply, int reply_len, + u8 *session_key, int *session_key_len, + u8 *con_secret, int *con_secret_len) +{ + struct ceph_mon_client *monc = con->private; + bool was_authed; + int ret; + + mutex_lock(&monc->mutex); + WARN_ON(!monc->hunting); + was_authed = ceph_auth_is_authenticated(monc->auth); + ret = ceph_auth_handle_reply_done(monc->auth, global_id, + reply, reply_len, + session_key, session_key_len, + con_secret, con_secret_len); + finish_auth(monc, ret, was_authed); + if (!ret) + finish_hunting(monc); + mutex_unlock(&monc->mutex); + return 0; +} + +static int mon_handle_auth_bad_method(struct ceph_connection *con, + int used_proto, int result, + const int *allowed_protos, int proto_cnt, + const int *allowed_modes, int mode_cnt) +{ + struct ceph_mon_client *monc = con->private; + bool was_authed; + + mutex_lock(&monc->mutex); + WARN_ON(!monc->hunting); + was_authed = ceph_auth_is_authenticated(monc->auth); + ceph_auth_handle_bad_method(monc->auth, used_proto, result, + allowed_protos, proto_cnt, + allowed_modes, mode_cnt); + finish_auth(monc, -EACCES, was_authed); + mutex_unlock(&monc->mutex); + return 0; +} + /* * handle incoming message */ @@ -1487,4 +1580,8 @@ static const struct ceph_connection_operations mon_con_ops = { .dispatch = dispatch, .fault = mon_fault, .alloc_msg = mon_alloc_msg, + .get_auth_request = mon_get_auth_request, + .handle_auth_reply_more = mon_handle_auth_reply_more, + .handle_auth_done = mon_handle_auth_done, + .handle_auth_bad_method = mon_handle_auth_bad_method, }; diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 51be5a7482fc..662b52e52651 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -3918,9 +3918,11 @@ static int handle_one_map(struct ceph_osd_client *osdc, set_pool_was_full(osdc); if (incremental) - newmap = osdmap_apply_incremental(&p, end, false, osdc->osdmap); + newmap = osdmap_apply_incremental(&p, end, + ceph_msgr2(osdc->client), + osdc->osdmap); else - newmap = ceph_osdmap_decode(&p, end, false); + newmap = ceph_osdmap_decode(&p, end, ceph_msgr2(osdc->client)); if (IS_ERR(newmap)) return PTR_ERR(newmap); @@ -5575,6 +5577,7 @@ static void put_osd_con(struct ceph_connection *con) /* * authentication */ + /* * Note: returned pointer is the address of a structure that's * managed separately. Caller must *not* attempt to free it. @@ -5640,6 +5643,80 @@ static int invalidate_authorizer(struct ceph_connection *con) return ceph_monc_validate_auth(&osdc->client->monc); } +static int osd_get_auth_request(struct ceph_connection *con, + void *buf, int *buf_len, + void **authorizer, int *authorizer_len) +{ + struct ceph_osd *o = con->private; + struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; + struct ceph_auth_handshake *auth = &o->o_auth; + int ret; + + ret = ceph_auth_get_authorizer(ac, auth, CEPH_ENTITY_TYPE_OSD, + buf, buf_len); + if (ret) + return ret; + + *authorizer = auth->authorizer_buf; + *authorizer_len = auth->authorizer_buf_len; + return 0; +} + +static int osd_handle_auth_reply_more(struct ceph_connection *con, + void *reply, int reply_len, + void *buf, int *buf_len, + void **authorizer, int *authorizer_len) +{ + struct ceph_osd *o = con->private; + struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; + struct ceph_auth_handshake *auth = &o->o_auth; + int ret; + + ret = ceph_auth_handle_svc_reply_more(ac, auth, reply, reply_len, + buf, buf_len); + if (ret) + return ret; + + *authorizer = auth->authorizer_buf; + *authorizer_len = auth->authorizer_buf_len; + return 0; +} + +static int osd_handle_auth_done(struct ceph_connection *con, + u64 global_id, void *reply, int reply_len, + u8 *session_key, int *session_key_len, + u8 *con_secret, int *con_secret_len) +{ + struct ceph_osd *o = con->private; + struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; + struct ceph_auth_handshake *auth = &o->o_auth; + + return ceph_auth_handle_svc_reply_done(ac, auth, reply, reply_len, + session_key, session_key_len, + con_secret, con_secret_len); +} + +static int osd_handle_auth_bad_method(struct ceph_connection *con, + int used_proto, int result, + const int *allowed_protos, int proto_cnt, + const int *allowed_modes, int mode_cnt) +{ + struct ceph_osd *o = con->private; + struct ceph_mon_client *monc = &o->o_osdc->client->monc; + int ret; + + if (ceph_auth_handle_bad_authorizer(monc->auth, CEPH_ENTITY_TYPE_OSD, + used_proto, result, + allowed_protos, proto_cnt, + allowed_modes, mode_cnt)) { + ret = ceph_monc_validate_auth(monc); + if (ret) + return ret; + } + + return -EACCES; +} + static void osd_reencode_message(struct ceph_msg *msg) { int type = le16_to_cpu(msg->hdr.type); @@ -5677,4 +5754,8 @@ static const struct ceph_connection_operations osd_con_ops = { .sign_message = osd_sign_message, .check_message_signature = osd_check_message_signature, .fault = osd_fault, + .get_auth_request = osd_get_auth_request, + .handle_auth_reply_more = osd_handle_auth_reply_more, + .handle_auth_done = osd_handle_auth_done, + .handle_auth_bad_method = osd_handle_auth_bad_method, }; |