greenplumn gpreader_test 源码
greenplumn gpreader_test 代码
文件路径:/gpcontrib/gpcloud/test/gpreader_test.cpp
#include "gpreader.cpp"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "mock_classes.h"
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Invoke;
using ::testing::Return;
using ::testing::Throw;
class MockGPReader : public GPReader {
public:
MockGPReader(const S3Params& params, S3RESTfulService* mockRESTfulService) : GPReader(params) {
restfulServicePtr = mockRESTfulService;
}
S3BucketReader& getBucketReader() {
return this->bucketReader;
}
};
class GPReaderTest : public testing::Test {
protected:
virtual void SetUp() {
eolString[0] = '\0';
}
virtual void TearDown() {
eolString[0] = '\n';
eolString[1] = '\0';
}
};
TEST_F(GPReaderTest, Construct) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
S3Params p = InitConfig(url + " config=data/s3test.conf");
EXPECT_EQ("secret_test", p.getCred().secret);
EXPECT_EQ("accessid_test", p.getCred().accessID);
EXPECT_EQ("ABCDEFGabcdefg", p.getCred().token);
}
TEST_F(GPReaderTest, Open) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
S3Params p(url);
MockS3RESTfulService mockRESTfulService(p);
MockGPReader gpreader(p, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response response(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _)).WillOnce(Return(response));
gpreader.open(p);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ((uint64_t)1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ((uint64_t)3, keyList.contents[0].getSize());
}
TEST_F(GPReaderTest, Close) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
S3Params p(url);
MockS3RESTfulService mockRESTfulService(p);
MockGPReader gpreader(p, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response response(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _)).WillOnce(Return(response));
gpreader.open(p);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ((uint64_t)1, keyList.contents.size());
gpreader.close();
EXPECT_TRUE(keyList.contents.empty());
}
TEST_F(GPReaderTest, ReadSmallData) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
S3Params p = InitConfig(url + " config=data/s3test.conf");
MockS3RESTfulService mockRESTfulService(p);
MockGPReader gpreader(p, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
vector<uint8_t> keyContent;
// generate 3 bytes in random way
srand(time(NULL));
keyContent.push_back(rand() & 0xFF);
keyContent.push_back(rand() & 0xFF);
keyContent.push_back(rand() & 0xFF);
Response keyReaderResponse(RESPONSE_OK, keyContent);
EXPECT_CALL(mockRESTfulService, get(_, _))
.WillOnce(Return(listBucketResponse))
// first 4 bytes is retrieved once for format detection.
.WillOnce(Return(keyReaderResponse))
// whole file content
.WillOnce(Return(keyReaderResponse));
gpreader.open(p);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
printf("Size: %lu\n", keyList.contents.size());
EXPECT_EQ((uint64_t)1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ((uint64_t)3, keyList.contents[0].getSize());
char buffer[64];
EXPECT_EQ((uint64_t)3, gpreader.read(buffer, sizeof(buffer)));
EXPECT_EQ(0, memcmp(buffer, keyContent.data(), 3));
EXPECT_EQ((uint64_t)0, gpreader.read(buffer, sizeof(buffer)));
}
// We need to mock the Get() to fulfill unordered GET request.
class MockS3RESTfulServiceForMultiThreads : public MockS3RESTfulService {
public:
MockS3RESTfulServiceForMultiThreads(const S3Params& params, uint64_t dataSize)
: MockS3RESTfulService(params) {
srand(time(NULL));
data.reserve(dataSize);
for (uint64_t i = 0; i < dataSize; i++) {
data.push_back(rand() & 0xFF);
}
}
Response mockGet(const string& url, HTTPHeaders& headers) {
string range = headers.Get(RANGE);
size_t index = range.find("=");
string rangeNumber = range.substr(index + 1);
index = rangeNumber.find("-");
// stoi is not available in C++98, use sscanf as workaround.
int begin, end;
if (index > 0) {
sscanf(rangeNumber.substr(0, index).c_str(), "%d", &begin);
} else {
begin = 0;
}
if (rangeNumber.empty()) {
end = data.size();
} else {
sscanf(rangeNumber.substr(index + 1).c_str(), "%d", &end);
}
vector<uint8_t> responseData(data.begin() + begin, data.begin() + end + 1);
return Response(RESPONSE_OK, responseData);
}
const uint8_t* getData() {
return data.data();
}
private:
vector<uint8_t> data;
};
TEST_F(GPReaderTest, ReadHugeData) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
S3Params p = InitConfig(url + " config=data/s3test.conf section=smallchunk");
// We don't know the chunksize before we load_config()
// so we create a big enough data (128M), to force
// multiple-threads to be constructed and loop-read to be triggered.
uint64_t totalData = 1024 * 1024 * 128;
MockS3RESTfulServiceForMultiThreads mockRESTfulService(p, totalData);
MockGPReader gpreader(p, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("bigdata/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("bigdata/", 0))
->pushBuckentContent(BucketContent("bigdata/bigdata", totalData));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
// call mockGet() instead of simply return a Response.
EXPECT_CALL(mockRESTfulService, get(_, _))
.WillOnce(Return(listBucketResponse))
.WillRepeatedly(Invoke(&mockRESTfulService, &MockS3RESTfulServiceForMultiThreads::mockGet));
gpreader.open(p);
// compare the data size
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ((uint64_t)1, keyList.contents.size());
EXPECT_EQ("bigdata/bigdata", keyList.contents[0].getName());
EXPECT_EQ(totalData, keyList.contents[0].getSize());
// compare the data content
static char buffer[1024 * 1024];
uint64_t size = sizeof(buffer);
for (uint64_t i = 0; i < totalData; i += size) {
EXPECT_EQ(size, gpreader.read(buffer, size));
ASSERT_EQ(0, memcmp(buffer, mockRESTfulService.getData() + i, size));
}
// Guarantee the last call
EXPECT_EQ((uint64_t)0, gpreader.read(buffer, sizeof(buffer)));
}
TEST_F(GPReaderTest, ReadFromEmptyURL) {
string url;
S3Params p(url);
MockS3RESTfulService mockRESTfulService(p);
MockGPReader gpreader(p, &mockRESTfulService);
// an exception should be throwed in parseURL()
// with message "'' is not valid"
EXPECT_THROW(gpreader.open(p), S3Exception);
}
/*
TEST_F(GPReaderTest, ReadFromInvalidURL) {
string baseUrl = "s3://";
S3Params p(baseUrl);
MockS3RESTfulService mockRESTfulService(p);
MockGPReader gpreader(p, &mockRESTfulService);
// an exception should be throwed in parseURL()
// with message "'s3://' is not valid,"
EXPECT_THROW(gpreader.open(p), S3Exception);
}*/
TEST_F(GPReaderTest, ReadAndGetFailedListBucketResponse) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
S3Params p(url);
MockS3RESTfulService mockRESTfulService(p);
MockGPReader gpreader(p, &mockRESTfulService);
EXPECT_CALL(mockRESTfulService, get(_, _)).WillRepeatedly(Throw(S3FailedAfterRetry("", 3, "")));
EXPECT_THROW(gpreader.open(p), S3FailedAfterRetry);
}
TEST_F(GPReaderTest, ReadAndGetFailedKeyReaderResponse) {
string url = "s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal";
S3Params p(url);
MockS3RESTfulService mockRESTfulService(p);
MockGPReader gpreader(p, &mockRESTfulService);
XMLGenerator generator;
XMLGenerator* gen = &generator;
gen->setName("s3test.pivotal.io")
->setPrefix("threebytes/")
->setIsTruncated(false)
->pushBuckentContent(BucketContent("threebytes/", 0))
->pushBuckentContent(BucketContent("threebytes/threebytes", 3));
Response listBucketResponse(RESPONSE_OK, gen->toXML());
EXPECT_CALL(mockRESTfulService, get(_, _))
.WillOnce(Return(listBucketResponse))
.WillRepeatedly(Throw(S3FailedAfterRetry("", 3, "")));
gpreader.open(p);
const ListBucketResult& keyList = gpreader.getBucketReader().getKeyList();
EXPECT_EQ((uint64_t)1, keyList.contents.size());
EXPECT_EQ("threebytes/threebytes", keyList.contents[0].getName());
EXPECT_EQ((uint64_t)3, keyList.contents[0].getSize());
char buffer[64];
EXPECT_THROW(gpreader.read(buffer, sizeof(buffer)), S3FailedAfterRetry);
}
// thread functions test with local variables
TEST(Common, ThreadFunctions) {
// just to test if these two are functional
thread_setup();
EXPECT_NE((void*)NULL, mutex_buf);
thread_cleanup();
EXPECT_EQ((void*)NULL, mutex_buf);
EXPECT_NE((uint64_t)0, gpcloud_id_function());
}
相关信息
相关文章
greenplumn compress_writer_test 源码
greenplumn decompress_reader_test 源码
greenplumn s3bucket_reader_test 源码
greenplumn s3common_reader_test 源码
greenplumn s3common_writer_test 源码
0
赞
热门推荐
-
2、 - 优质文章
-
3、 gate.io
-
8、 golang
-
9、 openharmony
-
10、 Vue中input框自动聚焦