1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
/*-
* Copyright (c) 2014-present MongoDB, Inc.
* Copyright (c) 2008-2014 WiredTiger, Inc.
* All rights reserved.
*
* See the file LICENSE for redistribution information.
*/
#include "wt_internal.h"
/*
* __tiered_confchk --
* Check for a valid tiered storage source.
*/
static int
__tiered_confchk(
WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name, WT_NAMED_STORAGE_SOURCE **nstoragep)
{
WT_CONNECTION_IMPL *conn;
WT_NAMED_STORAGE_SOURCE *nstorage;
*nstoragep = NULL;
if (name->len == 0 || WT_STRING_MATCH("none", name->str, name->len))
return (0);
conn = S2C(session);
TAILQ_FOREACH (nstorage, &conn->storagesrcqh, q)
if (WT_STRING_MATCH(nstorage->name, name->str, name->len)) {
*nstoragep = nstorage;
return (0);
}
WT_RET_MSG(session, EINVAL, "unknown storage source '%.*s'", (int)name->len, name->str);
}
/*
* __tiered_common_config --
* Parse configuration options common to connection and btrees.
*/
static int
__tiered_common_config(WT_SESSION_IMPL *session, const char **cfg, WT_BUCKET_STORAGE *bstorage)
{
WT_CONFIG_ITEM cval;
if (bstorage == NULL)
return (0);
WT_RET(__wt_config_gets(session, cfg, "tiered_storage.local_retention", &cval));
bstorage->retain_secs = (uint64_t)cval.val;
WT_RET(__wt_config_gets(session, cfg, "tiered_storage.object_target_size", &cval));
bstorage->object_size = (uint64_t)cval.val;
return (0);
}
/*
* __wt_tiered_bucket_config --
* Given a configuration, (re)configure the bucket storage and return that structure.
*/
int
__wt_tiered_bucket_config(
WT_SESSION_IMPL *session, const char *cfg[], WT_BUCKET_STORAGE **bstoragep)
{
WT_BUCKET_STORAGE *bstorage, *new;
WT_CONFIG_ITEM auth, bucket, cachedir, name, prefix;
WT_CONNECTION_IMPL *conn;
WT_DECL_ITEM(buf);
WT_DECL_RET;
WT_NAMED_STORAGE_SOURCE *nstorage;
WT_STORAGE_SOURCE *storage;
uint64_t hash_bucket, hash;
*bstoragep = NULL;
WT_RET(__wt_config_gets(session, cfg, "tiered_storage.name", &name));
WT_RET(__wt_scr_alloc(session, 0, &buf));
bstorage = new = NULL;
conn = S2C(session);
__wt_spin_lock(session, &conn->storage_lock);
WT_ERR(__tiered_confchk(session, &name, &nstorage));
if (nstorage == NULL) {
WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.bucket", &bucket));
if (bucket.len != 0)
WT_ERR_MSG(
session, EINVAL, "tiered_storage.bucket requires tiered_storage.name to be set");
goto done;
}
/*
* Check if tiered storage is set on the connection. If someone wants tiered storage on a table,
* it needs to be configured on the database as well.
*/
if (conn->bstorage == NULL && bstoragep != &conn->bstorage)
WT_ERR_MSG(
session, EINVAL, "table tiered storage requires connection tiered storage to be set");
/* A bucket and bucket_prefix are required, cache_directory and auth_token are not. */
WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.auth_token", &auth));
WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.bucket", &bucket));
if (bucket.len == 0)
WT_ERR_MSG(session, EINVAL, "table tiered storage requires bucket to be set");
WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.bucket_prefix", &prefix));
if (prefix.len == 0)
WT_ERR_MSG(session, EINVAL, "table tiered storage requires bucket_prefix to be set");
WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.cache_directory", &cachedir));
hash = __wt_hash_city64(bucket.str, bucket.len);
hash_bucket = hash & (conn->hash_size - 1);
TAILQ_FOREACH (bstorage, &nstorage->buckethashqh[hash_bucket], q) {
if (WT_STRING_MATCH(bstorage->bucket, bucket.str, bucket.len) &&
(WT_STRING_MATCH(bstorage->bucket_prefix, prefix.str, prefix.len))) {
*bstoragep = bstorage;
goto done;
}
}
WT_ERR(__wt_calloc_one(session, &new));
WT_ERR(__wt_strndup(session, auth.str, auth.len, &new->auth_token));
WT_ERR(__wt_strndup(session, bucket.str, bucket.len, &new->bucket));
WT_ERR(__wt_strndup(session, prefix.str, prefix.len, &new->bucket_prefix));
WT_ERR(__wt_strndup(session, cachedir.str, cachedir.len, &new->cache_directory));
storage = nstorage->storage_source;
if (cachedir.len != 0)
WT_ERR(__wt_buf_fmt(session, buf, "cache_directory=%s", new->cache_directory));
WT_ERR(storage->ss_customize_file_system(
storage, &session->iface, new->bucket, new->auth_token, buf->data, &new->file_system));
new->storage_source = storage;
/* If we're creating a new bucket storage, parse the other settings into it. */
TAILQ_INSERT_HEAD(&nstorage->bucketqh, new, q);
TAILQ_INSERT_HEAD(&nstorage->buckethashqh[hash_bucket], new, hashq);
F_SET(new, WT_BUCKET_FREE);
WT_ERR(__tiered_common_config(session, cfg, new));
*bstoragep = new;
done:
if (0) {
err:
if (new != NULL) {
__wt_free(session, new->bucket);
__wt_free(session, new->bucket_prefix);
}
__wt_free(session, new);
}
__wt_spin_unlock(session, &conn->storage_lock);
__wt_scr_free(session, &buf);
return (ret);
}
/*
* __wt_tiered_conn_config --
* Parse and setup the storage server options for the connection.
*/
int
__wt_tiered_conn_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfig)
{
WT_CONNECTION_IMPL *conn;
WT_DECL_RET;
conn = S2C(session);
if (!reconfig)
WT_RET(__wt_tiered_bucket_config(session, cfg, &conn->bstorage));
else
WT_ERR(__tiered_common_config(session, cfg, conn->bstorage));
/* If the connection is not set up for tiered storage there is nothing more to do. */
if (conn->bstorage == NULL)
return (0);
__wt_verbose(session, WT_VERB_TIERED, "TIERED_CONFIG: bucket %s", conn->bstorage->bucket);
__wt_verbose(
session, WT_VERB_TIERED, "TIERED_CONFIG: prefix %s", conn->bstorage->bucket_prefix);
WT_ASSERT(session, conn->bstorage != NULL);
WT_STAT_CONN_SET(session, tiered_object_size, conn->bstorage->object_size);
WT_STAT_CONN_SET(session, tiered_retention, conn->bstorage->retain_secs);
/*
* Set up the designated file system for the "none" bucket.
*/
WT_ASSERT(session, conn->file_system != NULL);
conn->bstorage_none.file_system = conn->file_system;
return (0);
err:
__wt_free(session, conn->bstorage->auth_token);
__wt_free(session, conn->bstorage->bucket);
__wt_free(session, conn->bstorage->bucket_prefix);
__wt_free(session, conn->bstorage->cache_directory);
__wt_free(session, conn->bstorage);
return (ret);
}
|
