thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

get rid of poco mongodb integration implementation

Browse Source
This commit is contained in:
Kirill Nikiforov 2024-11-16 00:28:54 +04:00
parent 8d4cc787fe
commit fcaa058c92
No known key found for this signature in database
104 changed files with 36 additions and 9219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
base/poco/CMakeLists.txt
View File

@ -3,11 +3,6 @@ add_subdirectory (Data)
add_subdirectory (Data/ODBC)
add_subdirectory (Foundation)
add_subdirectory (JSON)
if (USE_MONGODB)
add_subdirectory(MongoDB)
endif()
add_subdirectory (Net)
add_subdirectory (NetSSL_OpenSSL)
add_subdirectory (Redis)

16
base/poco/MongoDB/CMakeLists.txt
View File

@ -1,16 +0,0 @@
file (GLOB SRCS src/*.cpp)
add_library (_poco_mongodb ${SRCS})
add_library (Poco::MongoDB ALIAS _poco_mongodb)
# TODO: remove these warning exclusions
target_compile_options (_poco_mongodb
PRIVATE
-Wno-old-style-cast
-Wno-unused-parameter
-Wno-zero-as-null-pointer-constant
)
target_include_directories (_poco_mongodb SYSTEM PUBLIC "include")
target_link_libraries (_poco_mongodb PUBLIC Poco::Net)

142
base/poco/MongoDB/include/Poco/MongoDB/Array.h
View File

@ -1,142 +0,0 @@
//
// Array.h
//
// Library: MongoDB
// Package: MongoDB
// Module: Array
//
// Definition of the Array class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_Array_INCLUDED
#define MongoDB_Array_INCLUDED
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/NumberFormatter.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API Array : public Document
/// This class represents a BSON Array.
{
public:
using Ptr = SharedPtr<Array>;
Array();
/// Creates an empty Array.
virtual ~Array();
/// Destroys the Array.
// Document template functions available for backward compatibility
using Document::add;
using Document::get;
template <typename T>
Document & add(T value)
/// Creates an element with the name from the current pos and value and
/// adds it to the array document.
///
/// The active document is returned to allow chaining of the add methods.
{
return Document::add<T>(Poco::NumberFormatter::format(size()), value);
}
Document & add(const char * value)
/// Creates an element with a name from the current pos and value and
/// adds it to the array document.
///
/// The active document is returned to allow chaining of the add methods.
{
return Document::add(Poco::NumberFormatter::format(size()), value);
}
template <typename T>
T get(std::size_t pos) const
/// Returns the element at the given index and tries to convert
/// it to the template type. If the element is not found, a
/// Poco::NotFoundException will be thrown. If the element cannot be
/// converted a BadCastException will be thrown.
{
return Document::get<T>(Poco::NumberFormatter::format(pos));
}
template <typename T>
T get(std::size_t pos, const T & deflt) const
/// Returns the element at the given index and tries to convert
/// it to the template type. If the element is not found, or
/// has the wrong type, the deflt argument will be returned.
{
return Document::get<T>(Poco::NumberFormatter::format(pos), deflt);
}
Element::Ptr get(std::size_t pos) const;
/// Returns the element at the given index.
/// An empty element will be returned if the element is not found.
template <typename T>
bool isType(std::size_t pos) const
/// Returns true if the type of the element equals the TypeId of ElementTrait,
/// otherwise false.
{
return Document::isType<T>(Poco::NumberFormatter::format(pos));
}
std::string toString(int indent = 0) const;
/// Returns a string representation of the Array.
private:
friend void BSONReader::read<Array::Ptr>(Array::Ptr & to);
};
// BSON Embedded Array
// spec: document
template <>
struct ElementTraits<Array::Ptr>
{
enum
{
TypeId = 0x04
};
static std::string toString(const Array::Ptr & value, int indent = 0)
{
//TODO:
return value.isNull() ? "null" : value->toString(indent);
}
};
template <>
inline void BSONReader::read<Array::Ptr>(Array::Ptr & to)
{
to->read(_reader);
}
template <>
inline void BSONWriter::write<Array::Ptr>(Array::Ptr & from)
{
from->write(_writer);
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_Array_INCLUDED

88
base/poco/MongoDB/include/Poco/MongoDB/BSONReader.h
View File

@ -1,88 +0,0 @@
//
// BSONReader.h
//
// Library: MongoDB
// Package: MongoDB
// Module: BSONReader
//
// Definition of the BSONReader class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_BSONReader_INCLUDED
#define MongoDB_BSONReader_INCLUDED
#include "Poco/BinaryReader.h"
#include "Poco/MongoDB/MongoDB.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API BSONReader
/// Class for reading BSON using a Poco::BinaryReader
{
public:
BSONReader(const Poco::BinaryReader & reader) : _reader(reader)
/// Creates the BSONReader using the given BinaryWriter.
{
}
virtual ~BSONReader()
/// Destroys the BSONReader.
{
}
template <typename T>
void read(T & t)
/// Reads the value from the reader. The default implementation uses the >> operator to
/// the given argument. Special types can write their own version.
{
_reader >> t;
}
std::string readCString();
/// Reads a cstring from the reader.
/// A cstring is a string terminated with a 0x00.
private:
Poco::BinaryReader _reader;
};
//
// inlines
//
inline std::string BSONReader::readCString()
{
std::string val;
while (_reader.good())
{
char c;
_reader >> c;
if (_reader.good())
{
if (c == 0x00)
return val;
else
val += c;
}
}
return val;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_BSONReader_INCLUDED

76
base/poco/MongoDB/include/Poco/MongoDB/BSONWriter.h
View File

@ -1,76 +0,0 @@
//
// BSONWriter.h
//
// Library: MongoDB
// Package: MongoDB
// Module: BSONWriter
//
// Definition of the BSONWriter class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_BSONWriter_INCLUDED
#define MongoDB_BSONWriter_INCLUDED
#include "Poco/BinaryWriter.h"
#include "Poco/MongoDB/MongoDB.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API BSONWriter
/// Class for writing BSON using a Poco::BinaryWriter.
{
public:
BSONWriter(const Poco::BinaryWriter & writer) : _writer(writer)
/// Creates the BSONWriter.
{
}
virtual ~BSONWriter()
/// Destroys the BSONWriter.
{
}
template <typename T>
void write(T & t)
/// Writes the value to the writer. The default implementation uses
/// the << operator. Special types can write their own version.
{
_writer << t;
}
void writeCString(const std::string & value);
/// Writes a cstring to the writer. A cstring is a string
/// terminated a null character.
private:
Poco::BinaryWriter _writer;
};
//
// inlines
//
inline void BSONWriter::writeCString(const std::string & value)
{
_writer.writeRaw(value);
_writer << (unsigned char)0x00;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_BSONWriter_INCLUDED

158
base/poco/MongoDB/include/Poco/MongoDB/Binary.h
View File

@ -1,158 +0,0 @@
//
// Binary.h
//
// Library: MongoDB
// Package: MongoDB
// Module: Binary
//
// Definition of the Binary class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_Binary_INCLUDED
#define MongoDB_Binary_INCLUDED
#include <sstream>
#include "Poco/Base64Encoder.h"
#include "Poco/Buffer.h"
#include "Poco/MemoryStream.h"
#include "Poco/MongoDB/Element.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/StreamCopier.h"
#include "Poco/UUID.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API Binary
/// Implements BSON Binary.
///
/// A Binary stores its data in a Poco::Buffer<unsigned char>.
{
public:
using Ptr = SharedPtr<Binary>;
Binary();
/// Creates an empty Binary with subtype 0.
Binary(Poco::Int32 size, unsigned char subtype);
/// Creates a Binary with a buffer of the given size and the given subtype.
Binary(const UUID & uuid);
/// Creates a Binary containing an UUID.
Binary(const std::string & data, unsigned char subtype = 0);
/// Creates a Binary with the contents of the given string and the given subtype.
Binary(const void * data, Poco::Int32 size, unsigned char subtype = 0);
/// Creates a Binary with the contents of the given buffer and the given subtype.
virtual ~Binary();
/// Destroys the Binary.
Buffer<unsigned char> & buffer();
/// Returns a reference to the internal buffer
unsigned char subtype() const;
/// Returns the subtype.
void subtype(unsigned char type);
/// Sets the subtype.
std::string toString(int indent = 0) const;
/// Returns the contents of the Binary as Base64-encoded string.
std::string toRawString() const;
/// Returns the raw content of the Binary as a string.
UUID uuid() const;
/// Returns the UUID when the binary subtype is 0x04.
/// Otherwise, throws a Poco::BadCastException.
private:
Buffer<unsigned char> _buffer;
unsigned char _subtype;
};
//
// inlines
//
inline unsigned char Binary::subtype() const
{
return _subtype;
}
inline void Binary::subtype(unsigned char type)
{
_subtype = type;
}
inline Buffer<unsigned char> & Binary::buffer()
{
return _buffer;
}
inline std::string Binary::toRawString() const
{
return std::string(reinterpret_cast<const char *>(_buffer.begin()), _buffer.size());
}
// BSON Embedded Document
// spec: binary
template <>
struct ElementTraits<Binary::Ptr>
{
enum
{
TypeId = 0x05
};
static std::string toString(const Binary::Ptr & value, int indent = 0) { return value.isNull() ? "" : value->toString(); }
};
template <>
inline void BSONReader::read<Binary::Ptr>(Binary::Ptr & to)
{
Poco::Int32 size;
_reader >> size;
to->buffer().resize(size);
unsigned char subtype;
_reader >> subtype;
to->subtype(subtype);
_reader.readRaw((char *)to->buffer().begin(), size);
}
template <>
inline void BSONWriter::write<Binary::Ptr>(Binary::Ptr & from)
{
_writer << (Poco::Int32)from->buffer().size();
_writer << from->subtype();
_writer.writeRaw((char *)from->buffer().begin(), from->buffer().size());
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_Binary_INCLUDED

191
base/poco/MongoDB/include/Poco/MongoDB/Connection.h
View File

@ -1,191 +0,0 @@
//
// Connection.h
//
// Library: MongoDB
// Package: MongoDB
// Module: Connection
//
// Definition of the Connection class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_Connection_INCLUDED
#define MongoDB_Connection_INCLUDED
#include "Poco/MongoDB/OpMsgMessage.h"
#include "Poco/MongoDB/RequestMessage.h"
#include "Poco/MongoDB/ResponseMessage.h"
#include "Poco/Mutex.h"
#include "Poco/Net/SocketAddress.h"
#include "Poco/Net/StreamSocket.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API Connection
/// Represents a connection to a MongoDB server
/// using the MongoDB wire protocol.
///
/// See https://docs.mongodb.com/manual/reference/mongodb-wire-protocol/
/// for more information on the wire protocol.
{
public:
using Ptr = Poco::SharedPtr<Connection>;
class MongoDB_API SocketFactory
{
public:
SocketFactory();
/// Creates the SocketFactory.
virtual ~SocketFactory();
/// Destroys the SocketFactory.
virtual Poco::Net::StreamSocket createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure);
/// Creates a Poco::Net::StreamSocket (if secure is false), or a
/// Poco::Net::SecureStreamSocket (if secure is true) connected to the
/// given host and port number.
///
/// The default implementation will throw a Poco::NotImplementedException
/// if secure is true.
};
Connection();
/// Creates an unconnected Connection.
///
/// Use this when you want to connect later on.
Connection(const std::string & hostAndPort);
/// Creates a Connection connected to the given MongoDB instance at host:port.
///
/// The host and port must be separated with a colon.
Connection(const std::string & uri, SocketFactory & socketFactory);
/// Creates a Connection connected to the given MongoDB instance at the
/// given URI.
///
/// See the corresponding connect() method for more information.
Connection(const std::string & host, int port);
/// Creates a Connection connected to the given MongoDB instance at host and port.
Connection(const Poco::Net::SocketAddress & addrs);
/// Creates a Connection connected to the given MongoDB instance at the given address.
Connection(const Poco::Net::StreamSocket & socket);
/// Creates a Connection connected to the given MongoDB instance using the given socket,
/// which must already be connected.
virtual ~Connection();
/// Destroys the Connection.
Poco::Net::SocketAddress address() const;
/// Returns the address of the MongoDB server.
const std::string & uri() const;
/// Returns the uri on which the connection was made.
void connect(const std::string & hostAndPort);
/// Connects to the given MongoDB server.
///
/// The host and port must be separated with a colon.
void connect(const std::string & uri, SocketFactory & socketFactory);
/// Connects to the given MongoDB instance at the given URI.
///
/// The URI must be in standard MongoDB connection string URI format:
///
/// mongodb://<user>:<password>@hostname.com:<port>/database-name?options
///
/// The following options are supported:
///
/// - ssl: If ssl=true is specified, a custom SocketFactory subclass creating
/// a SecureStreamSocket must be supplied.
/// - connectTimeoutMS: Socket connection timeout in milliseconds.
/// - socketTimeoutMS: Socket send/receive timeout in milliseconds.
/// - authMechanism: Authentication mechanism. Only "SCRAM-SHA-1" (default)
/// and "MONGODB-CR" are supported.
///
/// Unknown options are silently ignored.
///
/// Will also attempt to authenticate using the specified credentials,
/// using Database::authenticate().
///
/// Throws a Poco::NoPermissionException if authentication fails.
void connect(const std::string & host, int port);
/// Connects to the given MongoDB server.
void connect(const Poco::Net::SocketAddress & addrs);
/// Connects to the given MongoDB server.
void connect(const Poco::Net::StreamSocket & socket);
/// Connects using an already connected socket.
void disconnect();
/// Disconnects from the MongoDB server.
void sendRequest(RequestMessage & request);
/// Sends a request to the MongoDB server.
///
/// Used for one-way requests without a response.
void sendRequest(RequestMessage & request, ResponseMessage & response);
/// Sends a request to the MongoDB server and receives the response.
///
/// Use this when a response is expected: only a "query" or "getmore"
/// request will return a response.
void sendRequest(OpMsgMessage & request, OpMsgMessage & response);
/// Sends a request to the MongoDB server and receives the response
/// using newer wire protocol with OP_MSG.
void sendRequest(OpMsgMessage & request);
/// Sends an unacknowledged request to the MongoDB server using newer
/// wire protocol with OP_MSG.
/// No response is sent by the server.
void readResponse(OpMsgMessage & response);
/// Reads additional response data when previous message's flag moreToCome
/// indicates that server will send more data.
/// NOTE: See comments in OpMsgCursor code.
protected:
void connect();
private:
Poco::Net::SocketAddress _address;
Poco::Net::StreamSocket _socket;
std::string _uri;
};
//
// inlines
//
inline Net::SocketAddress Connection::address() const
{
return _address;
}
inline const std::string & Connection::uri() const
{
return _uri;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_Connection_INCLUDED

80
base/poco/MongoDB/include/Poco/MongoDB/Cursor.h
View File

@ -1,80 +0,0 @@
//
// Cursor.h
//
// Library: MongoDB
// Package: MongoDB
// Module: Cursor
//
// Definition of the Cursor class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_Cursor_INCLUDED
#define MongoDB_Cursor_INCLUDED
#include "Poco/MongoDB/Connection.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/QueryRequest.h"
#include "Poco/MongoDB/ResponseMessage.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API Cursor : public Document
/// Cursor is an helper class for querying multiple documents.
{
public:
Cursor(const std::string & dbname, const std::string & collectionName, QueryRequest::Flags flags = QueryRequest::QUERY_DEFAULT);
/// Creates a Cursor for the given database and collection, using the specified flags.
Cursor(const std::string & fullCollectionName, QueryRequest::Flags flags = QueryRequest::QUERY_DEFAULT);
/// Creates a Cursor for the given database and collection ("database.collection"), using the specified flags.
Cursor(const Document & aggregationResponse);
/// Creates a Cursor for the given aggregation query response.
virtual ~Cursor();
/// Destroys the Cursor.
ResponseMessage & next(Connection & connection);
/// Tries to get the next documents. As long as ResponseMessage has a
/// cursor ID next can be called to retrieve the next bunch of documents.
///
/// The cursor must be killed (see kill()) when not all documents are needed.
QueryRequest & query();
/// Returns the associated query.
void kill(Connection & connection);
/// Kills the cursor and reset it so that it can be reused.
private:
QueryRequest _query;
ResponseMessage _response;
};
//
// inlines
//
inline QueryRequest & Cursor::query()
{
return _query;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_Cursor_INCLUDED

233
base/poco/MongoDB/include/Poco/MongoDB/Database.h
View File

@ -1,233 +0,0 @@
//
// Database.h
//
// Library: MongoDB
// Package: MongoDB
// Module: Database
//
// Definition of the Database class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_Database_INCLUDED
#define MongoDB_Database_INCLUDED
#include "Poco/MongoDB/Connection.h"
#include "Poco/MongoDB/DeleteRequest.h"
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/InsertRequest.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/QueryRequest.h"
#include "Poco/MongoDB/UpdateRequest.h"
#include "Poco/MongoDB/OpMsgCursor.h"
#include "Poco/MongoDB/OpMsgMessage.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API Database
/// Database is a helper class for creating requests. MongoDB works with
/// collection names and uses the part before the first dot as the name of
/// the database.
{
public:
explicit Database(const std::string & name);
/// Creates a Database for the database with the given name.
virtual ~Database();
/// Destroys the Database.
const std::string & name() const;
/// Database name
bool authenticate(
Connection & connection,
const std::string & username,
const std::string & password,
const std::string & method = AUTH_SCRAM_SHA1);
/// Authenticates against the database using the given connection,
/// username and password, as well as authentication method.
///
/// "MONGODB-CR" (default prior to MongoDB 3.0) and
/// "SCRAM-SHA-1" (default starting in 3.0) are the only supported
/// authentication methods.
///
/// Returns true if authentication was successful, otherwise false.
///
/// May throw a Poco::ProtocolException if authentication fails for a reason other than
/// invalid credentials.
Document::Ptr queryBuildInfo(Connection & connection) const;
/// Queries server build info (all wire protocols)
Document::Ptr queryServerHello(Connection & connection, bool old = false) const;
/// Queries hello response from server (all wire protocols)
Int64 count(Connection & connection, const std::string & collectionName) const;
/// Sends a count request for the given collection to MongoDB. (old wire protocol)
///
/// If the command fails, -1 is returned.
Poco::SharedPtr<Poco::MongoDB::QueryRequest> createCommand() const;
/// Creates a QueryRequest for a command. (old wire protocol)
Poco::SharedPtr<Poco::MongoDB::QueryRequest> createCountRequest(const std::string & collectionName) const;
/// Creates a QueryRequest to count the given collection.
/// The collectionname must not contain the database name. (old wire protocol)
Poco::SharedPtr<Poco::MongoDB::DeleteRequest> createDeleteRequest(const std::string & collectionName) const;
/// Creates a DeleteRequest to delete documents in the given collection.
/// The collectionname must not contain the database name. (old wire protocol)
Poco::SharedPtr<Poco::MongoDB::InsertRequest> createInsertRequest(const std::string & collectionName) const;
/// Creates an InsertRequest to insert new documents in the given collection.
/// The collectionname must not contain the database name. (old wire protocol)
Poco::SharedPtr<Poco::MongoDB::QueryRequest> createQueryRequest(const std::string & collectionName) const;
/// Creates a QueryRequest. (old wire protocol)
/// The collectionname must not contain the database name.
Poco::SharedPtr<Poco::MongoDB::UpdateRequest> createUpdateRequest(const std::string & collectionName) const;
/// Creates an UpdateRequest. (old wire protocol)
/// The collectionname must not contain the database name.
Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> createOpMsgMessage(const std::string & collectionName) const;
/// Creates OpMsgMessage. (new wire protocol)
Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> createOpMsgMessage() const;
/// Creates OpMsgMessage for database commands that do not require collection as an argument. (new wire protocol)
Poco::SharedPtr<Poco::MongoDB::OpMsgCursor> createOpMsgCursor(const std::string & collectionName) const;
/// Creates OpMsgCursor. (new wire protocol)
Poco::MongoDB::Document::Ptr ensureIndex(
Connection & connection,
const std::string & collection,
const std::string & indexName,
Poco::MongoDB::Document::Ptr keys,
bool unique = false,
bool background = false,
int version = 0,
int ttl = 0);
/// Creates an index. The document returned is the result of a getLastError call.
/// For more info look at the ensureIndex information on the MongoDB website. (old wire protocol)
Document::Ptr getLastErrorDoc(Connection & connection) const;
/// Sends the getLastError command to the database and returns the error document.
/// (old wire protocol)
std::string getLastError(Connection & connection) const;
/// Sends the getLastError command to the database and returns the err element
/// from the error document. When err is null, an empty string is returned.
/// (old wire protocol)
static const std::string AUTH_MONGODB_CR;
/// Default authentication mechanism prior to MongoDB 3.0.
static const std::string AUTH_SCRAM_SHA1;
/// Default authentication mechanism for MongoDB 3.0.
enum WireVersion
/// Wire version as reported by the command hello.
/// See details in MongoDB github, repository specifications.
/// @see queryServerHello
{
VER_26 = 1,
VER_26_2 = 2,
VER_30 = 3,
VER_32 = 4,
VER_34 = 5,
VER_36 = 6, ///< First wire version that supports OP_MSG
VER_40 = 7,
VER_42 = 8,
VER_44 = 9,
VER_50 = 13,
VER_51 = 14, ///< First wire version that supports only OP_MSG
VER_52 = 15,
VER_53 = 16,
VER_60 = 17
};
protected:
bool authCR(Connection & connection, const std::string & username, const std::string & password);
bool authSCRAM(Connection & connection, const std::string & username, const std::string & password);
private:
std::string _dbname;
};
//
// inlines
//
inline const std::string & Database::name() const
{
return _dbname;
}
inline Poco::SharedPtr<Poco::MongoDB::QueryRequest> Database::createCommand() const
{
Poco::SharedPtr<Poco::MongoDB::QueryRequest> cmd = createQueryRequest("$cmd");
cmd->setNumberToReturn(1);
return cmd;
}
inline Poco::SharedPtr<Poco::MongoDB::DeleteRequest> Database::createDeleteRequest(const std::string & collectionName) const
{
return new Poco::MongoDB::DeleteRequest(_dbname + '.' + collectionName);
}
inline Poco::SharedPtr<Poco::MongoDB::InsertRequest> Database::createInsertRequest(const std::string & collectionName) const
{
return new Poco::MongoDB::InsertRequest(_dbname + '.' + collectionName);
}
inline Poco::SharedPtr<Poco::MongoDB::QueryRequest> Database::createQueryRequest(const std::string & collectionName) const
{
return new Poco::MongoDB::QueryRequest(_dbname + '.' + collectionName);
}
inline Poco::SharedPtr<Poco::MongoDB::UpdateRequest> Database::createUpdateRequest(const std::string & collectionName) const
{
return new Poco::MongoDB::UpdateRequest(_dbname + '.' + collectionName);
}
// -- New wire protocol commands
inline Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> Database::createOpMsgMessage(const std::string & collectionName) const
{
return new Poco::MongoDB::OpMsgMessage(_dbname, collectionName);
}
inline Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> Database::createOpMsgMessage() const
{
// Collection name for database commands is not needed.
return createOpMsgMessage("");
}
inline Poco::SharedPtr<Poco::MongoDB::OpMsgCursor> Database::createOpMsgCursor(const std::string & collectionName) const
{
return new Poco::MongoDB::OpMsgCursor(_dbname, collectionName);
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_Database_INCLUDED

116
base/poco/MongoDB/include/Poco/MongoDB/DeleteRequest.h
View File

@ -1,116 +0,0 @@
//
// DeleteRequest.h
//
// Library: MongoDB
// Package: MongoDB
// Module: DeleteRequest
//
// Definition of the DeleteRequest class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_DeleteRequest_INCLUDED
#define MongoDB_DeleteRequest_INCLUDED
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/RequestMessage.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API DeleteRequest : public RequestMessage
/// A DeleteRequest is used to delete one or more documents from a database.
///
/// Specific flags for this request
/// - DELETE_DEFAULT: default delete operation
/// - DELETE_SINGLE_REMOVE: delete only the first document
{
public:
enum Flags
{
DELETE_DEFAULT = 0,
/// Default
DELETE_SINGLE_REMOVE = 1
/// Delete only the first document.
};
DeleteRequest(const std::string & collectionName, Flags flags = DELETE_DEFAULT);
/// Creates a DeleteRequest for the given collection using the given flags.
///
/// The full collection name is the concatenation of the database
/// name with the collection name, using a "." for the concatenation. For example,
/// for the database "foo" and the collection "bar", the full collection name is
/// "foo.bar".
DeleteRequest(const std::string & collectionName, bool justOne);
/// Creates a DeleteRequest for the given collection.
///
/// The full collection name is the concatenation of the database
/// name with the collection name, using a "." for the concatenation. For example,
/// for the database "foo" and the collection "bar", the full collection name is
/// "foo.bar".
///
/// If justOne is true, only the first matching document will
/// be removed (the same as using flag DELETE_SINGLE_REMOVE).
virtual ~DeleteRequest();
/// Destructor
Flags flags() const;
/// Returns the flags.
void flags(Flags flag);
/// Sets the flags.
Document & selector();
/// Returns the selector document.
protected:
void buildRequest(BinaryWriter & writer);
/// Writes the OP_DELETE request to the writer.
private:
Flags _flags;
std::string _fullCollectionName;
Document _selector;
};
///
/// inlines
///
inline DeleteRequest::Flags DeleteRequest::flags() const
{
return _flags;
}
inline void DeleteRequest::flags(DeleteRequest::Flags flags)
{
_flags = flags;
}
inline Document & DeleteRequest::selector()
{
return _selector;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_DeleteRequest_INCLUDED

296
base/poco/MongoDB/include/Poco/MongoDB/Document.h
View File

@ -1,296 +0,0 @@
//
// Document.h
//
// Library: MongoDB
// Package: MongoDB
// Module: Document
//
// Definition of the Document class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_Document_INCLUDED
#define MongoDB_Document_INCLUDED
#include <algorithm>
#include <cstdlib>
#include "Poco/BinaryReader.h"
#include "Poco/BinaryWriter.h"
#include "Poco/MongoDB/Element.h"
#include "Poco/MongoDB/MongoDB.h"
namespace Poco
{
namespace MongoDB
{
class Array;
class ElementFindByName
{
public:
ElementFindByName(const std::string & name) : _name(name) { }
bool operator()(const Element::Ptr & element) { return !element.isNull() && element->name() == _name; }
private:
std::string _name;
};
class MongoDB_API Document
/// Represents a MongoDB (BSON) document.
{
public:
using Ptr = SharedPtr<Document>;
using Vector = std::vector<Document::Ptr>;
Document();
/// Creates an empty Document.
virtual ~Document();
/// Destroys the Document.
Document & addElement(Element::Ptr element);
/// Add an element to the document.
///
/// The active document is returned to allow chaining of the add methods.
template <typename T>
Document & add(const std::string & name, T value)
/// Creates an element with the given name and value and
/// adds it to the document.
///
/// The active document is returned to allow chaining of the add methods.
{
return addElement(new ConcreteElement<T>(name, value));
}
Document & add(const std::string & name, const char * value)
/// Creates an element with the given name and value and
/// adds it to the document.
///
/// The active document is returned to allow chaining of the add methods.
{
return addElement(new ConcreteElement<std::string>(name, std::string(value)));
}
Document & addNewDocument(const std::string & name);
/// Create a new document and add it to this document.
/// Unlike the other add methods, this method returns
/// a reference to the new document.
Array & addNewArray(const std::string & name);
/// Create a new array and add it to this document.
/// Method returns a reference to the new array.
void clear();
/// Removes all elements from the document.
void elementNames(std::vector<std::string> & keys) const;
/// Puts all element names into std::vector.
bool empty() const;
/// Returns true if the document doesn't contain any documents.
bool exists(const std::string & name) const;
/// Returns true if the document has an element with the given name.
template <typename T>
T get(const std::string & name) const
/// Returns the element with the given name and tries to convert
/// it to the template type. When the element is not found, a
/// NotFoundException will be thrown. When the element can't be
/// converted a BadCastException will be thrown.
{
Element::Ptr element = get(name);
if (element.isNull())
{
throw NotFoundException(name);
}
else
{
if (ElementTraits<T>::TypeId == element->type())
{
ConcreteElement<T> * concrete = dynamic_cast<ConcreteElement<T> *>(element.get());
if (concrete != 0)
{
return concrete->value();
}
}
throw BadCastException("Invalid type mismatch!");
}
}
template <typename T>
T get(const std::string & name, const T & def) const
/// Returns the element with the given name and tries to convert
/// it to the template type. When the element is not found, or
/// has the wrong type, the def argument will be returned.
{
Element::Ptr element = get(name);
if (element.isNull())
{
return def;
}
if (ElementTraits<T>::TypeId == element->type())
{
ConcreteElement<T> * concrete = dynamic_cast<ConcreteElement<T> *>(element.get());
if (concrete != 0)
{
return concrete->value();
}
}
return def;
}
Element::Ptr get(const std::string & name) const;
/// Returns the element with the given name.
/// An empty element will be returned when the element is not found.
Int64 getInteger(const std::string & name) const;
/// Returns an integer. Useful when MongoDB returns Int32, Int64
/// or double for a number (count for example). This method will always
/// return an Int64. When the element is not found, a
/// Poco::NotFoundException will be thrown.
bool remove(const std::string & name);
/// Removes an element from the document.
template <typename T>
bool isType(const std::string & name) const
/// Returns true when the type of the element equals the TypeId of ElementTrait.
{
Element::Ptr element = get(name);
if (element.isNull())
{
return false;
}
return ElementTraits<T>::TypeId == element->type();
}
void read(BinaryReader & reader);
/// Reads a document from the reader
std::size_t size() const;
/// Returns the number of elements in the document.
virtual std::string toString(int indent = 0) const;
/// Returns a String representation of the document.
void write(BinaryWriter & writer);
/// Writes a document to the reader
protected:
ElementSet _elements;
};
//
// inlines
//
inline Document & Document::addElement(Element::Ptr element)
{
_elements.push_back(element);
return *this;
}
inline Document & Document::addNewDocument(const std::string & name)
{
Document::Ptr newDoc = new Document();
add(name, newDoc);
return *newDoc;
}
inline void Document::clear()
{
_elements.clear();
}
inline bool Document::empty() const
{
return _elements.empty();
}
inline void Document::elementNames(std::vector<std::string> & keys) const
{
for (ElementSet::const_iterator it = _elements.begin(); it != _elements.end(); ++it)
{
keys.push_back((*it)->name());
}
}
inline bool Document::exists(const std::string & name) const
{
return std::find_if(_elements.begin(), _elements.end(), ElementFindByName(name)) != _elements.end();
}
inline bool Document::remove(const std::string & name)
{
auto it = std::find_if(_elements.begin(), _elements.end(), ElementFindByName(name));
if (it == _elements.end())
return false;
_elements.erase(it);
return true;
}
inline std::size_t Document::size() const
{
return _elements.size();
}
// BSON Embedded Document
// spec: document
template <>
struct ElementTraits<Document::Ptr>
{
enum
{
TypeId = 0x03
};
static std::string toString(const Document::Ptr & value, int indent = 0)
{
return value.isNull() ? "null" : value->toString(indent);
}
};
template <>
inline void BSONReader::read<Document::Ptr>(Document::Ptr & to)
{
to->read(_reader);
}
template <>
inline void BSONWriter::write<Document::Ptr>(Document::Ptr & from)
{
from->write(_writer);
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_Document_INCLUDED

393
base/poco/MongoDB/include/Poco/MongoDB/Element.h
View File

@ -1,393 +0,0 @@
//
// Element.h
//
// Library: MongoDB
// Package: MongoDB
// Module: Element
//
// Definition of the Element class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_Element_INCLUDED
#define MongoDB_Element_INCLUDED
#include <iomanip>
#include <list>
#include <sstream>
#include <string>
#include "Poco/BinaryReader.h"
#include "Poco/BinaryWriter.h"
#include "Poco/DateTimeFormatter.h"
#include "Poco/MongoDB/BSONReader.h"
#include "Poco/MongoDB/BSONWriter.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/Nullable.h"
#include "Poco/NumberFormatter.h"
#include "Poco/SharedPtr.h"
#include "Poco/Timestamp.h"
#include "Poco/UTF8String.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API Element
/// Represents an Element of a Document or an Array.
{
public:
using Ptr = Poco::SharedPtr<Element>;
explicit Element(const std::string & name);
/// Creates the Element with the given name.
virtual ~Element();
/// Destructor
const std::string & name() const;
/// Returns the name of the element.
virtual std::string toString(int indent = 0) const = 0;
/// Returns a string representation of the element.
virtual int type() const = 0;
/// Returns the MongoDB type of the element.
private:
virtual void read(BinaryReader & reader) = 0;
virtual void write(BinaryWriter & writer) = 0;
friend class Document;
std::string _name;
};
//
// inlines
//
inline const std::string & Element::name() const
{
return _name;
}
using ElementSet = std::list<Element::Ptr>;
template <typename T>
struct ElementTraits
{
};
// BSON Floating point
// spec: double
template <>
struct ElementTraits<double>
{
enum
{
TypeId = 0x01
};
static std::string toString(const double & value, int indent = 0) { return Poco::NumberFormatter::format(value); }
};
// BSON UTF-8 string
// spec: int32 (byte*) "\x00"
// int32 is the number bytes in byte* + 1 (for trailing "\x00")
template <>
struct ElementTraits<std::string>
{
enum
{
TypeId = 0x02
};
static std::string toString(const std::string & value, int indent = 0)
{
std::ostringstream oss;
oss << '"';
for (std::string::const_iterator it = value.begin(); it != value.end(); ++it)
{
switch (*it)
{
case '"':
oss << "\\\"";
break;
case '\\':
oss << "\\\\";
break;
case '\b':
oss << "\\b";
break;
case '\f':
oss << "\\f";
break;
case '\n':
oss << "\\n";
break;
case '\r':
oss << "\\r";
break;
case '\t':
oss << "\\t";
break;
default: {
if (*it > 0 && *it <= 0x1F)
{
oss << "\\u" << std::hex << std::uppercase << std::setfill('0') << std::setw(4) << static_cast<int>(*it);
}
else
{
oss << *it;
}
break;
}
}
}
oss << '"';
return oss.str();
}
};
template <>
inline void BSONReader::read<std::string>(std::string & to)
{
Poco::Int32 size;
_reader >> size;
_reader.readRaw(size, to);
to.erase(to.end() - 1); // remove terminating 0
}
template <>
inline void BSONWriter::write<std::string>(std::string & from)
{
_writer << (Poco::Int32)(from.length() + 1);
writeCString(from);
}
// BSON bool
// spec: "\x00" "\x01"
template <>
struct ElementTraits<bool>
{
enum
{
TypeId = 0x08
};
static std::string toString(const bool & value, int indent = 0) { return value ? "true" : "false"; }
};
template <>
inline void BSONReader::read<bool>(bool & to)
{
unsigned char b;
_reader >> b;
to = b != 0;
}
template <>
inline void BSONWriter::write<bool>(bool & from)
{
unsigned char b = from ? 0x01 : 0x00;
_writer << b;
}
// BSON 32-bit integer
// spec: int32
template <>
struct ElementTraits<Int32>
{
enum
{
TypeId = 0x10
};
static std::string toString(const Int32 & value, int indent = 0) { return Poco::NumberFormatter::format(value); }
};
// BSON UTC datetime
// spec: int64
template <>
struct ElementTraits<Timestamp>
{
enum
{
TypeId = 0x09
};
static std::string toString(const Timestamp & value, int indent = 0)
{
std::string result;
result.append(1, '"');
result.append(DateTimeFormatter::format(value, "%Y-%m-%dT%H:%M:%s%z"));
result.append(1, '"');
return result;
}
};
template <>
inline void BSONReader::read<Timestamp>(Timestamp & to)
{
Poco::Int64 value;
_reader >> value;
to = Timestamp::fromEpochTime(static_cast<std::time_t>(value / 1000));
to += (value % 1000 * 1000);
}
template <>
inline void BSONWriter::write<Timestamp>(Timestamp & from)
{
_writer << (from.epochMicroseconds() / 1000);
}
using NullValue = Nullable<unsigned char>;
// BSON Null Value
// spec:
template <>
struct ElementTraits<NullValue>
{
enum
{
TypeId = 0x0A
};
static std::string toString(const NullValue & value, int indent = 0) { return "null"; }
};
template <>
inline void BSONReader::read<NullValue>(NullValue & to)
{
}
template <>
inline void BSONWriter::write<NullValue>(NullValue & from)
{
}
struct BSONTimestamp
{
Poco::Timestamp ts;
Poco::Int32 inc;
};
// BSON Timestamp
// spec: int64
template <>
struct ElementTraits<BSONTimestamp>
{
enum
{
TypeId = 0x11
};
static std::string toString(const BSONTimestamp & value, int indent = 0)
{
std::string result;
result.append(1, '"');
result.append(DateTimeFormatter::format(value.ts, "%Y-%m-%dT%H:%M:%s%z"));
result.append(1, ' ');
result.append(NumberFormatter::format(value.inc));
result.append(1, '"');
return result;
}
};
template <>
inline void BSONReader::read<BSONTimestamp>(BSONTimestamp & to)
{
Poco::Int64 value;
_reader >> value;
to.inc = value & 0xffffffff;
value >>= 32;
to.ts = Timestamp::fromEpochTime(static_cast<std::time_t>(value));
}
template <>
inline void BSONWriter::write<BSONTimestamp>(BSONTimestamp & from)
{
Poco::Int64 value = from.ts.epochMicroseconds() / 1000;
value <<= 32;
value += from.inc;
_writer << value;
}
// BSON 64-bit integer
// spec: int64
template <>
struct ElementTraits<Int64>
{
enum
{
TypeId = 0x12
};
static std::string toString(const Int64 & value, int indent = 0) { return NumberFormatter::format(value); }
};
template <typename T>
class ConcreteElement : public Element
{
public:
ConcreteElement(const std::string & name, const T & init) : Element(name), _value(init) { }
virtual ~ConcreteElement() { }
T value() const { return _value; }
std::string toString(int indent = 0) const { return ElementTraits<T>::toString(_value, indent); }
int type() const { return ElementTraits<T>::TypeId; }
void read(BinaryReader & reader) { BSONReader(reader).read(_value); }
void write(BinaryWriter & writer) { BSONWriter(writer).write(_value); }
private:
T _value;
};
}
} // namespace Poco::MongoDB
#endif // MongoDB_Element_INCLUDED

92
base/poco/MongoDB/include/Poco/MongoDB/GetMoreRequest.h
View File

@ -1,92 +0,0 @@
//
// GetMoreRequest.h
//
// Library: MongoDB
// Package: MongoDB
// Module: GetMoreRequest
//
// Definition of the GetMoreRequest class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_GetMoreRequest_INCLUDED
#define MongoDB_GetMoreRequest_INCLUDED
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/RequestMessage.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API GetMoreRequest : public RequestMessage
/// A GetMoreRequest is used to query the database for more documents in a collection
/// after a query request is send (OP_GETMORE).
{
public:
GetMoreRequest(const std::string & collectionName, Int64 cursorID);
/// Creates a GetMoreRequest for the give collection and cursor.
///
/// The full collection name is the concatenation of the database
/// name with the collection name, using a "." for the concatenation. For example,
/// for the database "foo" and the collection "bar", the full collection name is
/// "foo.bar". The cursorID has been returned by the response on the query request.
/// By default the numberToReturn is set to 100.
virtual ~GetMoreRequest();
/// Destroys the GetMoreRequest.
Int32 getNumberToReturn() const;
/// Returns the limit of returned documents.
void setNumberToReturn(Int32 n);
/// Sets the limit of returned documents.
Int64 cursorID() const;
/// Returns the cursor ID.
protected:
void buildRequest(BinaryWriter & writer);
private:
std::string _fullCollectionName;
Int32 _numberToReturn;
Int64 _cursorID;
};
//
// inlines
//
inline Int32 GetMoreRequest::getNumberToReturn() const
{
return _numberToReturn;
}
inline void GetMoreRequest::setNumberToReturn(Int32 n)
{
_numberToReturn = n;
}
inline Int64 GetMoreRequest::cursorID() const
{
return _cursorID;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_GetMoreRequest_INCLUDED

100
base/poco/MongoDB/include/Poco/MongoDB/InsertRequest.h
View File

@ -1,100 +0,0 @@
//
// InsertRequest.h
//
// Library: MongoDB
// Package: MongoDB
// Module: InsertRequest
//
// Definition of the InsertRequest class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_InsertRequest_INCLUDED
#define MongoDB_InsertRequest_INCLUDED
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/RequestMessage.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API InsertRequest : public RequestMessage
/// A request for inserting one or more documents to the database
/// (OP_INSERT).
{
public:
enum Flags
{
INSERT_DEFAULT = 0,
/// If specified, perform a normal insert operation.
INSERT_CONTINUE_ON_ERROR = 1
/// If set, the database will not stop processing a bulk insert if one
/// fails (e.g. due to duplicate IDs). This makes bulk insert behave similarly
/// to a series of single inserts, except lastError will be set if any insert
/// fails, not just the last one. If multiple errors occur, only the most
/// recent will be reported.
};
InsertRequest(const std::string & collectionName, Flags flags = INSERT_DEFAULT);
/// Creates an InsertRequest.
///
/// The full collection name is the concatenation of the database
/// name with the collection name, using a "." for the concatenation. For example,
/// for the database "foo" and the collection "bar", the full collection name is
/// "foo.bar".
virtual ~InsertRequest();
/// Destroys the InsertRequest.
Document & addNewDocument();
/// Adds a new document for insertion. A reference to the empty document is
/// returned. InsertRequest is the owner of the Document and will free it
/// on destruction.
Document::Vector & documents();
/// Returns the documents to insert into the database.
protected:
void buildRequest(BinaryWriter & writer);
private:
Int32 _flags;
std::string _fullCollectionName;
Document::Vector _documents;
};
//
// inlines
//
inline Document & InsertRequest::addNewDocument()
{
Document::Ptr doc = new Document();
_documents.push_back(doc);
return *doc;
}
inline Document::Vector & InsertRequest::documents()
{
return _documents;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_InsertRequest_INCLUDED

108
base/poco/MongoDB/include/Poco/MongoDB/JavaScriptCode.h
View File

@ -1,108 +0,0 @@
//
// JavaScriptCode.h
//
// Library: MongoDB
// Package: MongoDB
// Module: JavaScriptCode
//
// Definition of the JavaScriptCode class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_JavaScriptCode_INCLUDED
#define MongoDB_JavaScriptCode_INCLUDED
#include "Poco/MongoDB/BSONReader.h"
#include "Poco/MongoDB/BSONWriter.h"
#include "Poco/MongoDB/Element.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/SharedPtr.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API JavaScriptCode
/// Represents JavaScript type in BSON.
{
public:
using Ptr = SharedPtr<JavaScriptCode>;
JavaScriptCode();
/// Creates an empty JavaScriptCode object.
virtual ~JavaScriptCode();
/// Destroys the JavaScriptCode.
void setCode(const std::string & code);
/// Sets the JavaScript code.
std::string getCode() const;
/// Returns the JavaScript code.
private:
std::string _code;
};
//
// inlines
//
inline void JavaScriptCode::setCode(const std::string & code)
{
_code = code;
}
inline std::string JavaScriptCode::getCode() const
{
return _code;
}
// BSON JavaScript code
// spec: string
template <>
struct ElementTraits<JavaScriptCode::Ptr>
{
enum
{
TypeId = 0x0D
};
static std::string toString(const JavaScriptCode::Ptr & value, int indent = 0) { return value.isNull() ? "" : value->getCode(); }
};
template <>
inline void BSONReader::read<JavaScriptCode::Ptr>(JavaScriptCode::Ptr & to)
{
std::string code;
BSONReader(_reader).read(code);
to = new JavaScriptCode();
to->setCode(code);
}
template <>
inline void BSONWriter::write<JavaScriptCode::Ptr>(JavaScriptCode::Ptr & from)
{
std::string code = from->getCode();
BSONWriter(_writer).write(code);
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_JavaScriptCode_INCLUDED

65
base/poco/MongoDB/include/Poco/MongoDB/KillCursorsRequest.h
View File

@ -1,65 +0,0 @@
//
// KillCursorsRequest.h
//
// Library: MongoDB
// Package: MongoDB
// Module: KillCursorsRequest
//
// Definition of the KillCursorsRequest class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_KillCursorsRequest_INCLUDED
#define MongoDB_KillCursorsRequest_INCLUDED
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/RequestMessage.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API KillCursorsRequest : public RequestMessage
/// Class for creating an OP_KILL_CURSORS client request. This
/// request is used to kill cursors, which are still open,
/// returned by query requests.
{
public:
KillCursorsRequest();
/// Creates a KillCursorsRequest.
virtual ~KillCursorsRequest();
/// Destroys the KillCursorsRequest.
std::vector<Int64> & cursors();
/// The internal list of cursors.
protected:
void buildRequest(BinaryWriter & writer);
std::vector<Int64> _cursors;
};
//
// inlines
//
inline std::vector<Int64> & KillCursorsRequest::cursors()
{
return _cursors;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_KillCursorsRequest_INCLUDED

76
base/poco/MongoDB/include/Poco/MongoDB/Message.h
View File

@ -1,76 +0,0 @@
//
// Message.h
//
// Library: MongoDB
// Package: MongoDB
// Module: Message
//
// Definition of the Message class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_Message_INCLUDED
#define MongoDB_Message_INCLUDED
#include <sstream>
#include "Poco/BinaryReader.h"
#include "Poco/BinaryWriter.h"
#include "Poco/MongoDB/MessageHeader.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/Net/Socket.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API Message
/// Base class for all messages send or retrieved from MongoDB server.
{
public:
explicit Message(MessageHeader::OpCode opcode);
/// Creates a Message using the given OpCode.
virtual ~Message();
/// Destructor
MessageHeader & header();
/// Returns the message header
protected:
MessageHeader _header;
void messageLength(Poco::Int32 length);
/// Sets the message length in the message header
};
//
// inlines
//
inline MessageHeader & Message::header()
{
return _header;
}
inline void Message::messageLength(Poco::Int32 length)
{
poco_assert(length > 0);
_header.setMessageLength(length);
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_Message_INCLUDED

140
base/poco/MongoDB/include/Poco/MongoDB/MessageHeader.h
View File

@ -1,140 +0,0 @@
//
// MessageHeader.h
//
// Library: MongoDB
// Package: MongoDB
// Module: MessageHeader
//
// Definition of the MessageHeader class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_MessageHeader_INCLUDED
#define MongoDB_MessageHeader_INCLUDED
#include "Poco/MongoDB/MessageHeader.h"
#include "Poco/MongoDB/MongoDB.h"
namespace Poco
{
namespace MongoDB
{
class Message; // Required to disambiguate friend declaration in MessageHeader.
class MongoDB_API MessageHeader
/// Represents the message header which is always prepended to a
/// MongoDB request or response message.
{
public:
static const unsigned int MSG_HEADER_SIZE = 16;
enum OpCode
{
// Opcodes deprecated in MongoDB 5.0
OP_REPLY = 1,
OP_UPDATE = 2001,
OP_INSERT = 2002,
OP_QUERY = 2004,
OP_GET_MORE = 2005,
OP_DELETE = 2006,
OP_KILL_CURSORS = 2007,
/// Opcodes supported in MongoDB 5.1 and later
OP_COMPRESSED = 2012,
OP_MSG = 2013
};
explicit MessageHeader(OpCode);
/// Creates the MessageHeader using the given OpCode.
virtual ~MessageHeader();
/// Destroys the MessageHeader.
void read(BinaryReader & reader);
/// Reads the header using the given BinaryReader.
void write(BinaryWriter & writer);
/// Writes the header using the given BinaryWriter.
Int32 getMessageLength() const;
/// Returns the message length.
OpCode opCode() const;
/// Returns the OpCode.
Int32 getRequestID() const;
/// Returns the request ID of the current message.
void setRequestID(Int32 id);
/// Sets the request ID of the current message.
Int32 responseTo() const;
/// Returns the request id from the original request.
private:
void setMessageLength(Int32 length);
/// Sets the message length.
Int32 _messageLength;
Int32 _requestID;
Int32 _responseTo;
OpCode _opCode;
friend class Message;
};
//
// inlines
//
inline MessageHeader::OpCode MessageHeader::opCode() const
{
return _opCode;
}
inline Int32 MessageHeader::getMessageLength() const
{
return _messageLength;
}
inline void MessageHeader::setMessageLength(Int32 length)
{
poco_assert(_messageLength >= 0);
_messageLength = MSG_HEADER_SIZE + length;
}
inline void MessageHeader::setRequestID(Int32 id)
{
_requestID = id;
}
inline Int32 MessageHeader::getRequestID() const
{
return _requestID;
}
inline Int32 MessageHeader::responseTo() const
{
return _responseTo;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_MessageHeader_INCLUDED

64
base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h
View File

@ -1,64 +0,0 @@
//
// MongoDB.h
//
// Library: MongoDB
// Package: MongoDB
// Module: MongoDB
//
// Basic definitions for the Poco MongoDB library.
// This file must be the first file included by every other MongoDB
// header file.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDBMongoDB_INCLUDED
#define MongoDBMongoDB_INCLUDED
#include "Poco/Foundation.h"
//
// The following block is the standard way of creating macros which make exporting
// from a DLL simpler. All files within this DLL are compiled with the MongoDB_EXPORTS
// symbol defined on the command line. this symbol should not be defined on any project
// that uses this DLL. This way any other project whose source files include this file see
// MongoDB_API functions as being imported from a DLL, whereas this DLL sees symbols
// defined with this macro as being exported.
//
#if defined(_WIN32) && defined(POCO_DLL)
# if defined(MongoDB_EXPORTS)
# define MongoDB_API __declspec(dllexport)
# else
# define MongoDB_API __declspec(dllimport)
# endif
#endif
#if !defined(MongoDB_API)
# if !defined(POCO_NO_GCC_API_ATTRIBUTE) && defined(__GNUC__) && (__GNUC__ >= 4)
# define MongoDB_API __attribute__((visibility("default")))
# else
# define MongoDB_API
# endif
#endif
//
// Automatically link MongoDB library.
//
#if defined(_MSC_VER)
# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(MongoDB_EXPORTS)
# pragma comment(lib, "PocoMongoDB" POCO_LIB_SUFFIX)
# endif
#endif
#endif // MongoDBMongoDB_INCLUDED

151
base/poco/MongoDB/include/Poco/MongoDB/ObjectId.h
View File

@ -1,151 +0,0 @@
//
// Array.h
//
// Library: MongoDB
// Package: MongoDB
// Module: ObjectId
//
// Definition of the ObjectId class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_ObjectId_INCLUDED
#define MongoDB_ObjectId_INCLUDED
#include "Poco/MongoDB/Element.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/Timestamp.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API ObjectId
/// ObjectId is a 12-byte BSON type, constructed using:
///
/// - a 4-byte timestamp,
/// - a 3-byte machine identifier,
/// - a 2-byte process id, and
/// - a 3-byte counter, starting with a random value.
///
/// In MongoDB, documents stored in a collection require a unique _id field that acts
/// as a primary key. Because ObjectIds are small, most likely unique, and fast to generate,
/// MongoDB uses ObjectIds as the default value for the _id field if the _id field is not
/// specified; i.e., the mongod adds the _id field and generates a unique ObjectId to assign
/// as its value.
{
public:
using Ptr = SharedPtr<ObjectId>;
explicit ObjectId(const std::string & id);
/// Creates an ObjectId from a string.
///
/// The string must contain a hexadecimal representation
/// of an object ID. This means a string of 24 characters.
ObjectId(const ObjectId & copy);
/// Creates an ObjectId by copying another one.
virtual ~ObjectId();
/// Destroys the ObjectId.
Timestamp timestamp() const;
/// Returns the timestamp which is stored in the first four bytes of the id
std::string toString(const std::string & fmt = "%02x") const;
/// Returns the id in string format. The fmt parameter
/// specifies the formatting used for individual members
/// of the ID char array.
private:
ObjectId();
static int fromHex(char c);
static char fromHex(const char * c);
unsigned char _id[12];
friend class BSONWriter;
friend class BSONReader;
friend class Document;
};
//
// inlines
//
inline Timestamp ObjectId::timestamp() const
{
int time;
char * T = (char *)&time;
T[0] = _id[3];
T[1] = _id[2];
T[2] = _id[1];
T[3] = _id[0];
return Timestamp::fromEpochTime((time_t)time);
}
inline int ObjectId::fromHex(char c)
{
if ('0' <= c && c <= '9')
return c - '0';
if ('a' <= c && c <= 'f')
return c - 'a' + 10;
if ('A' <= c && c <= 'F')
return c - 'A' + 10;
return 0xff;
}
inline char ObjectId::fromHex(const char * c)
{
return (char)((fromHex(c[0]) << 4) | fromHex(c[1]));
}
// BSON Embedded Document
// spec: ObjectId
template <>
struct ElementTraits<ObjectId::Ptr>
{
enum
{
TypeId = 0x07
};
static std::string toString(const ObjectId::Ptr & id, int indent = 0, const std::string & fmt = "%02x")
{
return id->toString(fmt);
}
};
template <>
inline void BSONReader::read<ObjectId::Ptr>(ObjectId::Ptr & to)
{
_reader.readRaw((char *)to->_id, 12);
}
template <>
inline void BSONWriter::write<ObjectId::Ptr>(ObjectId::Ptr & from)
{
_writer.writeRaw((char *)from->_id, 12);
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_ObjectId_INCLUDED

96
base/poco/MongoDB/include/Poco/MongoDB/OpMsgCursor.h
View File

@ -1,96 +0,0 @@
//
// OpMsgCursor.h
//
// Library: MongoDB
// Package: MongoDB
// Module: OpMsgCursor
//
// Definition of the OpMsgCursor class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_OpMsgCursor_INCLUDED
#define MongoDB_OpMsgCursor_INCLUDED
#include "Poco/MongoDB/Connection.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/OpMsgMessage.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API OpMsgCursor : public Document
/// OpMsgCursor is an helper class for querying multiple documents using OpMsgMessage.
{
public:
OpMsgCursor(const std::string & dbname, const std::string & collectionName);
/// Creates a OpMsgCursor for the given database and collection.
virtual ~OpMsgCursor();
/// Destroys the OpMsgCursor.
void setEmptyFirstBatch(bool empty);
/// Empty first batch is used to get error response faster with little server processing
bool emptyFirstBatch() const;
void setBatchSize(Int32 batchSize);
/// Set non-default batch size
Int32 batchSize() const;
/// Current batch size (zero or negative number indicates default batch size)
Int64 cursorID() const;
OpMsgMessage & next(Connection & connection);
/// Tries to get the next documents. As long as response message has a
/// cursor ID next can be called to retrieve the next bunch of documents.
///
/// The cursor must be killed (see kill()) when not all documents are needed.
OpMsgMessage & query();
/// Returns the associated query.
void kill(Connection & connection);
/// Kills the cursor and reset it so that it can be reused.
private:
OpMsgMessage _query;
OpMsgMessage _response;
bool _emptyFirstBatch{false};
Int32 _batchSize{-1};
/// Batch size used in the cursor. Zero or negative value means that default shall be used.
Int64 _cursorID{0};
};
//
// inlines
//
inline OpMsgMessage & OpMsgCursor::query()
{
return _query;
}
inline Int64 OpMsgCursor::cursorID() const
{
return _cursorID;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_OpMsgCursor_INCLUDED

163
base/poco/MongoDB/include/Poco/MongoDB/OpMsgMessage.h
View File

@ -1,163 +0,0 @@
//
// OpMsgMessage.h
//
// Library: MongoDB
// Package: MongoDB
// Module: OpMsgMessage
//
// Definition of the OpMsgMessage class.
//
// Copyright (c) 2022, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_OpMsgMessage_INCLUDED
#define MongoDB_OpMsgMessage_INCLUDED
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/Message.h"
#include "Poco/MongoDB/MongoDB.h"
#include <string>
namespace Poco
{
namespace MongoDB
{
class MongoDB_API OpMsgMessage : public Message
/// This class represents a request/response (OP_MSG) to send requests and receive responses to/from MongoDB.
{
public:
// Constants for most often used MongoDB commands that can be sent using OP_MSG
// For complete list see: https://www.mongodb.com/docs/manual/reference/command/
// Query and write
static const std::string CMD_INSERT;
static const std::string CMD_DELETE;
static const std::string CMD_UPDATE;
static const std::string CMD_FIND;
static const std::string CMD_FIND_AND_MODIFY;
static const std::string CMD_GET_MORE;
// Aggregation
static const std::string CMD_AGGREGATE;
static const std::string CMD_COUNT;
static const std::string CMD_DISTINCT;
static const std::string CMD_MAP_REDUCE;
// Replication and administration
static const std::string CMD_HELLO;
static const std::string CMD_REPL_SET_GET_STATUS;
static const std::string CMD_REPL_SET_GET_CONFIG;
static const std::string CMD_CREATE;
static const std::string CMD_CREATE_INDEXES;
static const std::string CMD_DROP;
static const std::string CMD_DROP_DATABASE;
static const std::string CMD_KILL_CURSORS;
static const std::string CMD_LIST_DATABASES;
static const std::string CMD_LIST_INDEXES;
// Diagnostic
static const std::string CMD_BUILD_INFO;
static const std::string CMD_COLL_STATS;
static const std::string CMD_DB_STATS;
static const std::string CMD_HOST_INFO;
enum Flags : UInt32
{
MSG_FLAGS_DEFAULT = 0,
MSG_CHECKSUM_PRESENT = (1 << 0),
MSG_MORE_TO_COME = (1 << 1),
/// Sender will send another message and is not prepared for overlapping messages
MSG_EXHAUST_ALLOWED = (1 << 16)
/// Client is prepared for multiple replies (using the moreToCome bit) to this request
};
OpMsgMessage();
/// Creates an OpMsgMessage for response.
OpMsgMessage(const std::string & databaseName, const std::string & collectionName, UInt32 flags = MSG_FLAGS_DEFAULT);
/// Creates an OpMsgMessage for requests.
virtual ~OpMsgMessage();
const std::string & databaseName() const;
const std::string & collectionName() const;
void setCommandName(const std::string & command);
/// Sets the command name and clears the command document
void setCursor(Poco::Int64 cursorID, Poco::Int32 batchSize = -1);
/// Sets the command "getMore" for the cursor id with batch size (if it is not negative).
const std::string & commandName() const;
/// Current command name.
void setAcknowledgedRequest(bool ack);
/// Set false to create request that does not return response.
/// It has effect only for commands that write or delete documents.
/// Default is true (request returns acknowledge response).
bool acknowledgedRequest() const;
UInt32 flags() const;
Document & body();
/// Access to body document.
/// Additional query arguments shall be added after setting the command name.
const Document & body() const;
Document::Vector & documents();
/// Documents prepared for request or retrieved in response.
const Document::Vector & documents() const;
/// Documents prepared for request or retrieved in response.
bool responseOk() const;
/// Reads "ok" status from the response message.
void clear();
/// Clears the message.
void send(std::ostream & ostr);
/// Writes the request to stream.
void read(std::istream & istr);
/// Reads the response from the stream.
private:
enum PayloadType : UInt8
{
PAYLOAD_TYPE_0 = 0,
PAYLOAD_TYPE_1 = 1
};
std::string _databaseName;
std::string _collectionName;
UInt32 _flags{MSG_FLAGS_DEFAULT};
std::string _commandName;
bool _acknowledged{true};
Document _body;
Document::Vector _documents;
};
}
} // namespace Poco::MongoDB
#endif // MongoDB_OpMsgMessage_INCLUDED

123
base/poco/MongoDB/include/Poco/MongoDB/PoolableConnectionFactory.h
View File

@ -1,123 +0,0 @@
//
// PoolableConnectionFactory.h
//
// Library: MongoDB
// Package: MongoDB
// Module: PoolableConnectionFactory
//
// Definition of the PoolableConnectionFactory class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_PoolableConnectionFactory_INCLUDED
#define MongoDB_PoolableConnectionFactory_INCLUDED
#include "Poco/MongoDB/Connection.h"
#include "Poco/ObjectPool.h"
namespace Poco
{
template <>
class PoolableObjectFactory<MongoDB::Connection, MongoDB::Connection::Ptr>
/// PoolableObjectFactory specialisation for Connection. New connections
/// are created with the given address or URI.
///
/// If a Connection::SocketFactory is given, it must live for the entire
/// lifetime of the PoolableObjectFactory.
{
public:
PoolableObjectFactory(Net::SocketAddress & address) : _address(address), _pSocketFactory(0) { }
PoolableObjectFactory(const std::string & address) : _address(address), _pSocketFactory(0) { }
PoolableObjectFactory(const std::string & uri, MongoDB::Connection::SocketFactory & socketFactory)
: _uri(uri), _pSocketFactory(&socketFactory)
{
}
MongoDB::Connection::Ptr createObject()
{
if (_pSocketFactory)
return new MongoDB::Connection(_uri, *_pSocketFactory);
else
return new MongoDB::Connection(_address);
}
bool validateObject(MongoDB::Connection::Ptr pObject) { return true; }
void activateObject(MongoDB::Connection::Ptr pObject) { }
void deactivateObject(MongoDB::Connection::Ptr pObject) { }
void destroyObject(MongoDB::Connection::Ptr pObject) { }
private:
Net::SocketAddress _address;
std::string _uri;
MongoDB::Connection::SocketFactory * _pSocketFactory;
};
namespace MongoDB
{
class PooledConnection
/// Helper class for borrowing and returning a connection automatically from a pool.
{
public:
PooledConnection(Poco::ObjectPool<Connection, Connection::Ptr> & pool) : _pool(pool) { _connection = _pool.borrowObject(); }
virtual ~PooledConnection()
{
try
{
if (_connection)
{
_pool.returnObject(_connection);
}
}
catch (...)
{
poco_unexpected();
}
}
operator Connection::Ptr() { return _connection; }
#if defined(POCO_ENABLE_CPP11)
// Disable copy to prevent unwanted release of resources: C++11 way
PooledConnection(const PooledConnection &) = delete;
PooledConnection & operator=(const PooledConnection &) = delete;
// Enable move semantics
PooledConnection(PooledConnection && other) = default;
PooledConnection & operator=(PooledConnection &&) = default;
#endif
private:
#if !defined(POCO_ENABLE_CPP11)
// Disable copy to prevent unwanted release of resources: pre C++11 way
PooledConnection(const PooledConnection &);
PooledConnection & operator=(const PooledConnection &);
#endif
Poco::ObjectPool<Connection, Connection::Ptr> & _pool;
Connection::Ptr _connection;
};
}
} // namespace Poco::MongoDB
#endif // MongoDB_PoolableConnectionFactory_INCLUDED

190
base/poco/MongoDB/include/Poco/MongoDB/QueryRequest.h
View File

@ -1,190 +0,0 @@
//
// QueryRequest.h
//
// Library: MongoDB
// Package: MongoDB
// Module: QueryRequest
//
// Definition of the QueryRequest class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_QueryRequest_INCLUDED
#define MongoDB_QueryRequest_INCLUDED
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/RequestMessage.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API QueryRequest : public RequestMessage
/// A request to query documents in a MongoDB database
/// using an OP_QUERY request.
{
public:
enum Flags
{
QUERY_DEFAULT = 0,
/// Do not set any flags.
QUERY_TAILABLE_CURSOR = 2,
/// Tailable means cursor is not closed when the last data is retrieved.
/// Rather, the cursor marks the final object’s position.
/// You can resume using the cursor later, from where it was located,
/// if more data were received. Like any "latent cursor", the cursor may
/// become invalid at some point (CursorNotFound) – for example if the final
/// object it references were deleted.
QUERY_SLAVE_OK = 4,
/// Allow query of replica slave. Normally these return an error except
/// for namespace "local".
// QUERY_OPLOG_REPLAY = 8 (internal replication use only - drivers should not implement)
QUERY_NO_CURSOR_TIMEOUT = 16,
/// The server normally times out idle cursors after an inactivity period
/// (10 minutes) to prevent excess memory use. Set this option to prevent that.
QUERY_AWAIT_DATA = 32,
/// Use with QUERY_TAILABLECURSOR. If we are at the end of the data, block for
/// a while rather than returning no data. After a timeout period, we do
/// return as normal.
QUERY_EXHAUST = 64,
/// Stream the data down full blast in multiple "more" packages, on the
/// assumption that the client will fully read all data queried.
/// Faster when you are pulling a lot of data and know you want to pull
/// it all down.
/// Note: the client is not allowed to not read all the data unless it
/// closes the connection.
QUERY_PARTIAL = 128
/// Get partial results from a mongos if some shards are down
/// (instead of throwing an error).
};
QueryRequest(const std::string & collectionName, Flags flags = QUERY_DEFAULT);
/// Creates a QueryRequest.
///
/// The full collection name is the concatenation of the database
/// name with the collection name, using a "." for the concatenation. For example,
/// for the database "foo" and the collection "bar", the full collection name is
/// "foo.bar".
virtual ~QueryRequest();
/// Destroys the QueryRequest.
Flags getFlags() const;
/// Returns the flags.
void setFlags(Flags flag);
/// Set the flags.
std::string fullCollectionName() const;
/// Returns the <db>.<collection> used for this query.
Int32 getNumberToSkip() const;
/// Returns the number of documents to skip.
void setNumberToSkip(Int32 n);
/// Sets the number of documents to skip.
Int32 getNumberToReturn() const;
/// Returns the number of documents to return.
void setNumberToReturn(Int32 n);
/// Sets the number of documents to return (limit).
Document & selector();
/// Returns the selector document.
Document & returnFieldSelector();
/// Returns the field selector document.
protected:
void buildRequest(BinaryWriter & writer);
private:
Flags _flags;
std::string _fullCollectionName;
Int32 _numberToSkip;
Int32 _numberToReturn;
Document _selector;
Document _returnFieldSelector;
};
//
// inlines
//
inline QueryRequest::Flags QueryRequest::getFlags() const
{
return _flags;
}
inline void QueryRequest::setFlags(QueryRequest::Flags flags)
{
_flags = flags;
}
inline std::string QueryRequest::fullCollectionName() const
{
return _fullCollectionName;
}
inline Document & QueryRequest::selector()
{
return _selector;
}
inline Document & QueryRequest::returnFieldSelector()
{
return _returnFieldSelector;
}
inline Int32 QueryRequest::getNumberToSkip() const
{
return _numberToSkip;
}
inline void QueryRequest::setNumberToSkip(Int32 n)
{
_numberToSkip = n;
}
inline Int32 QueryRequest::getNumberToReturn() const
{
return _numberToReturn;
}
inline void QueryRequest::setNumberToReturn(Int32 n)
{
_numberToReturn = n;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_QueryRequest_INCLUDED

135
base/poco/MongoDB/include/Poco/MongoDB/RegularExpression.h
View File

@ -1,135 +0,0 @@
//
// RegularExpression.h
//
// Library: MongoDB
// Package: MongoDB
// Module: RegularExpression
//
// Definition of the RegularExpression class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_RegularExpression_INCLUDED
#define MongoDB_RegularExpression_INCLUDED
#include "Poco/MongoDB/Element.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/RegularExpression.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API RegularExpression
/// Represents a regular expression in BSON format.
{
public:
using Ptr = SharedPtr<RegularExpression>;
RegularExpression();
/// Creates an empty RegularExpression.
RegularExpression(const std::string & pattern, const std::string & options);
/// Creates a RegularExpression using the given pattern and options.
virtual ~RegularExpression();
/// Destroys the RegularExpression.
SharedPtr<Poco::RegularExpression> createRE() const;
/// Tries to create a Poco::RegularExpression from the MongoDB regular expression.
std::string getOptions() const;
/// Returns the options string.
void setOptions(const std::string & options);
/// Sets the options string.
std::string getPattern() const;
/// Returns the pattern.
void setPattern(const std::string & pattern);
/// Sets the pattern.
private:
std::string _pattern;
std::string _options;
};
///
/// inlines
///
inline std::string RegularExpression::getPattern() const
{
return _pattern;
}
inline void RegularExpression::setPattern(const std::string & pattern)
{
_pattern = pattern;
}
inline std::string RegularExpression::getOptions() const
{
return _options;
}
inline void RegularExpression::setOptions(const std::string & options)
{
_options = options;
}
// BSON Regex
// spec: cstring cstring
template <>
struct ElementTraits<RegularExpression::Ptr>
{
enum
{
TypeId = 0x0B
};
static std::string toString(const RegularExpression::Ptr & value, int indent = 0)
{
//TODO
return "RE: not implemented yet";
}
};
template <>
inline void BSONReader::read<RegularExpression::Ptr>(RegularExpression::Ptr & to)
{
std::string pattern = readCString();
std::string options = readCString();
to = new RegularExpression(pattern, options);
}
template <>
inline void BSONWriter::write<RegularExpression::Ptr>(RegularExpression::Ptr & from)
{
writeCString(from->getPattern());
writeCString(from->getOptions());
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_RegularExpression_INCLUDED

61
base/poco/MongoDB/include/Poco/MongoDB/ReplicaSet.h
View File

@ -1,61 +0,0 @@
//
// ReplicaSet.h
//
// Library: MongoDB
// Package: MongoDB
// Module: ReplicaSet
//
// Definition of the ReplicaSet class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_ReplicaSet_INCLUDED
#define MongoDB_ReplicaSet_INCLUDED
#include <vector>
#include "Poco/MongoDB/Connection.h"
#include "Poco/Net/SocketAddress.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API ReplicaSet
/// Class for working with a MongoDB replica set.
{
public:
explicit ReplicaSet(const std::vector<Net::SocketAddress> & addresses);
/// Creates the ReplicaSet using the given server addresses.
virtual ~ReplicaSet();
/// Destroys the ReplicaSet.
Connection::Ptr findMaster();
/// Tries to find the master MongoDB instance from the addresses
/// passed to the constructor.
///
/// Returns the Connection to the master, or null if no master
/// instance was found.
protected:
Connection::Ptr isMaster(const Net::SocketAddress & host);
private:
std::vector<Net::SocketAddress> _addresses;
};
}
} // namespace Poco::MongoDB
#endif // MongoDB_ReplicaSet_INCLUDED

54
base/poco/MongoDB/include/Poco/MongoDB/RequestMessage.h
View File

@ -1,54 +0,0 @@
//
// RequestMessage.h
//
// Library: MongoDB
// Package: MongoDB
// Module: RequestMessage
//
// Definition of the RequestMessage class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_RequestMessage_INCLUDED
#define MongoDB_RequestMessage_INCLUDED
#include <ostream>
#include "Poco/MongoDB/Message.h"
#include "Poco/MongoDB/MongoDB.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API RequestMessage : public Message
/// Base class for a request sent to the MongoDB server.
{
public:
explicit RequestMessage(MessageHeader::OpCode opcode);
/// Creates a RequestMessage using the given opcode.
virtual ~RequestMessage();
/// Destroys the RequestMessage.
void send(std::ostream & ostr);
/// Writes the request to stream.
protected:
virtual void buildRequest(BinaryWriter & ss) = 0;
};
}
} // namespace Poco::MongoDB
#endif // MongoDB_RequestMessage_INCLUDED

114
base/poco/MongoDB/include/Poco/MongoDB/ResponseMessage.h
View File

@ -1,114 +0,0 @@
//
// ResponseMessage.h
//
// Library: MongoDB
// Package: MongoDB
// Module: ResponseMessage
//
// Definition of the ResponseMessage class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_ResponseMessage_INCLUDED
#define MongoDB_ResponseMessage_INCLUDED
#include <cstdlib>
#include <istream>
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/Message.h"
#include "Poco/MongoDB/MongoDB.h"
namespace Poco
{
namespace MongoDB
{
class MongoDB_API ResponseMessage : public Message
/// This class represents a response (OP_REPLY) from MongoDB.
{
public:
ResponseMessage();
/// Creates an empty ResponseMessage.
ResponseMessage(const Int64 & cursorID);
/// Creates an ResponseMessage for existing cursor ID.
virtual ~ResponseMessage();
/// Destroys the ResponseMessage.
Int64 cursorID() const;
/// Returns the cursor ID.
void clear();
/// Clears the response.
std::size_t count() const;
/// Returns the number of documents in the response.
Document::Vector & documents();
/// Returns a vector containing the received documents.
bool empty() const;
/// Returns true if the response does not contain any documents.
bool hasDocuments() const;
/// Returns true if there is at least one document in the response.
void read(std::istream & istr);
/// Reads the response from the stream.
private:
Int32 _responseFlags;
Int64 _cursorID;
Int32 _startingFrom;
Int32 _numberReturned;
Document::Vector _documents;
};
//
// inlines
//
inline std::size_t ResponseMessage::count() const
{
return _documents.size();
}
inline bool ResponseMessage::empty() const
{
return _documents.size() == 0;
}
inline Int64 ResponseMessage::cursorID() const
{
return _cursorID;
}
inline Document::Vector & ResponseMessage::documents()
{
return _documents;
}
inline bool ResponseMessage::hasDocuments() const
{
return _documents.size() > 0;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_ResponseMessage_INCLUDED

117
base/poco/MongoDB/include/Poco/MongoDB/UpdateRequest.h
View File

@ -1,117 +0,0 @@
//
// UpdateRequest.h
//
// Library: MongoDB
// Package: MongoDB
// Module: UpdateRequest
//
// Definition of the UpdateRequest class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef MongoDB_UpdateRequest_INCLUDED
#define MongoDB_UpdateRequest_INCLUDED
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/MongoDB.h"
#include "Poco/MongoDB/RequestMessage.h"
namespace Poco
{
namespace MongoDB
{
class UpdateRequest : public RequestMessage
/// This request is used to update a document in a database
/// using the OP_UPDATE client request.
{
public:
enum Flags
{
UPDATE_DEFAULT = 0,
/// If set, the database will insert the supplied object into the
/// collection if no matching document is found.
UPDATE_UPSERT = 1,
/// If set, the database will update all matching objects in the collection.
/// Otherwise only updates first matching doc.
UPDATE_MULTIUPDATE = 2
/// If set to, updates multiple documents that meet the query criteria.
/// Otherwise only updates one document.
};
UpdateRequest(const std::string & collectionName, Flags flags = UPDATE_DEFAULT);
/// Creates the UpdateRequest.
///
/// The full collection name is the concatenation of the database
/// name with the collection name, using a "." for the concatenation. For example,
/// for the database "foo" and the collection "bar", the full collection name is
/// "foo.bar".
virtual ~UpdateRequest();
/// Destroys the UpdateRequest.
Document & selector();
/// Returns the selector document.
Document & update();
/// Returns the document to update.
Flags flags() const;
/// Returns the flags
void flags(Flags flags);
/// Sets the flags
protected:
void buildRequest(BinaryWriter & writer);
private:
Flags _flags;
std::string _fullCollectionName;
Document _selector;
Document _update;
};
//
// inlines
//
inline UpdateRequest::Flags UpdateRequest::flags() const
{
return _flags;
}
inline void UpdateRequest::flags(UpdateRequest::Flags flags)
{
_flags = flags;
}
inline Document & UpdateRequest::selector()
{
return _selector;
}
inline Document & UpdateRequest::update()
{
return _update;
}
}
} // namespace Poco::MongoDB
#endif // MongoDB_UpdateRequest_INCLUDED

75
base/poco/MongoDB/src/Array.cpp
View File

@ -1,75 +0,0 @@
//
// Array.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: Array
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/Array.h"
#include <sstream>
namespace Poco {
namespace MongoDB {
Array::Array():
Document()
{
}
Array::~Array()
{
}
Element::Ptr Array::get(std::size_t pos) const
{
std::string name = Poco::NumberFormatter::format(pos);
return Document::get(name);
}
std::string Array::toString(int indent) const
{
std::ostringstream oss;
oss << "[";
if (indent > 0) oss << std::endl;
for (ElementSet::const_iterator it = _elements.begin(); it != _elements.end(); ++it)
{
if (it != _elements.begin())
{
oss << ",";
if (indent > 0) oss << std::endl;
}
for (int i = 0; i < indent; ++i) oss << ' ';
oss << (*it)->toString(indent > 0 ? indent + 2 : 0);
}
if (indent > 0)
{
oss << std::endl;
if (indent >= 2) indent -= 2;
for (int i = 0; i < indent; ++i) oss << ' ';
}
oss << "]";
return oss.str();
}
} } // Namespace Poco::Mongo

89
base/poco/MongoDB/src/Binary.cpp
View File

@ -1,89 +0,0 @@
//
// Binary.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: Binary
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/Binary.h"
namespace Poco {
namespace MongoDB {
Binary::Binary():
_buffer(0),
_subtype(0)
{
}
Binary::Binary(Poco::Int32 size, unsigned char subtype):
_buffer(size),
_subtype(subtype)
{
}
Binary::Binary(const UUID& uuid):
_buffer(128 / 8),
_subtype(0x04)
{
unsigned char szUUID[16];
uuid.copyTo((char*) szUUID);
_buffer.assign(szUUID, 16);
}
Binary::Binary(const std::string& data, unsigned char subtype):
_buffer(reinterpret_cast<const unsigned char*>(data.data()), data.size()),
_subtype(subtype)
{
}
Binary::Binary(const void* data, Poco::Int32 size, unsigned char subtype):
_buffer(reinterpret_cast<const unsigned char*>(data), size),
_subtype(subtype)
{
}
Binary::~Binary()
{
}
std::string Binary::toString(int indent) const
{
std::ostringstream oss;
Base64Encoder encoder(oss);
MemoryInputStream mis((const char*) _buffer.begin(), _buffer.size());
StreamCopier::copyStream(mis, encoder);
encoder.close();
return oss.str();
}
UUID Binary::uuid() const
{
if ((_subtype == 0x04 || _subtype == 0x03) && _buffer.size() == 16)
{
UUID uuid;
uuid.copyFrom((const char*) _buffer.begin());
return uuid;
}
throw BadCastException("Invalid subtype: " + std::to_string(_subtype) + ", size: " + std::to_string(_buffer.size()));
}
} } // namespace Poco::MongoDB

348
base/poco/MongoDB/src/Connection.cpp
View File

@ -1,348 +0,0 @@
//
// Connection.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: Connection
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/Net/SocketStream.h"
#include "Poco/MongoDB/Connection.h"
#include "Poco/MongoDB/Database.h"
#include "Poco/URI.h"
#include "Poco/Format.h"
#include "Poco/NumberParser.h"
#include "Poco/Exception.h"
namespace Poco {
namespace MongoDB {
Connection::SocketFactory::SocketFactory()
{
}
Connection::SocketFactory::~SocketFactory()
{
}
Poco::Net::StreamSocket Connection::SocketFactory::createSocket(const std::string& host, int port, Poco::Timespan connectTimeout, bool secure)
{
if (!secure)
{
Poco::Net::SocketAddress addr(host, port);
Poco::Net::StreamSocket socket;
if (connectTimeout > 0)
socket.connect(addr, connectTimeout);
else
socket.connect(addr);
return socket;
}
else throw Poco::NotImplementedException("Default SocketFactory implementation does not support SecureStreamSocket");
}
Connection::Connection():
_address(),
_socket()
{
}
Connection::Connection(const std::string& hostAndPort):
_address(hostAndPort),
_socket()
{
connect();
}
Connection::Connection(const std::string& uri, SocketFactory& socketFactory):
_address(),
_socket()
{
connect(uri, socketFactory);
}
Connection::Connection(const std::string& host, int port):
_address(host, port),
_socket()
{
connect();
}
Connection::Connection(const Poco::Net::SocketAddress& addrs):
_address(addrs),
_socket()
{
connect();
}
Connection::Connection(const Poco::Net::StreamSocket& socket):
_address(socket.peerAddress()),
_socket(socket)
{
}
Connection::~Connection()
{
try
{
disconnect();
}
catch (...)
{
}
}
void Connection::connect()
{
_socket.connect(_address);
}
void Connection::connect(const std::string& hostAndPort)
{
_address = Poco::Net::SocketAddress(hostAndPort);
connect();
}
void Connection::connect(const std::string& host, int port)
{
_address = Poco::Net::SocketAddress(host, port);
connect();
}
void Connection::connect(const Poco::Net::SocketAddress& addrs)
{
_address = addrs;
connect();
}
void Connection::connect(const Poco::Net::StreamSocket& socket)
{
_address = socket.peerAddress();
_socket = socket;
}
void Connection::connect(const std::string& uri, SocketFactory& socketFactory)
{
std::vector<std::string> strAddresses;
std::string newURI;
if (uri.find(',') != std::string::npos)
{
size_t pos;
size_t head = 0;
if ((pos = uri.find("@")) != std::string::npos)
{
head = pos + 1;
}
else if ((pos = uri.find("://")) != std::string::npos)
{
head = pos + 3;
}
std::string tempstr;
std::string::const_iterator it = uri.begin();
it += head;
size_t tail = head;
for (;it != uri.end() && *it != '?' && *it != '/'; ++it)
{
tempstr += *it;
tail++;
}
it = tempstr.begin();
std::string token;
for (;it != tempstr.end(); ++it)
{
if (*it == ',')
{
newURI = uri.substr(0, head) + token + uri.substr(tail, uri.length());
strAddresses.push_back(newURI);
token = "";
}
else
{
token += *it;
}
}
newURI = uri.substr(0, head) + token + uri.substr(tail, uri.length());
strAddresses.push_back(newURI);
}
else
{
strAddresses.push_back(uri);
}
newURI = strAddresses.front();
Poco::URI theURI(newURI);
if (theURI.getScheme() != "mongodb") throw Poco::UnknownURISchemeException(uri);
std::string userInfo = theURI.getUserInfo();
std::string databaseName = theURI.getPath();
if (!databaseName.empty() && databaseName[0] == '/') databaseName.erase(0, 1);
if (databaseName.empty()) databaseName = "admin";
bool ssl = false;
Poco::Timespan connectTimeout;
Poco::Timespan socketTimeout;
std::string authMechanism = Database::AUTH_SCRAM_SHA1;
std::string readPreference="primary";
Poco::URI::QueryParameters params = theURI.getQueryParameters();
for (Poco::URI::QueryParameters::const_iterator it = params.begin(); it != params.end(); ++it)
{
if (it->first == "ssl")
{
ssl = (it->second == "true");
}
else if (it->first == "connectTimeoutMS")
{
connectTimeout = static_cast<Poco::Timespan::TimeDiff>(1000)*Poco::NumberParser::parse(it->second);
}
else if (it->first == "socketTimeoutMS")
{
socketTimeout = static_cast<Poco::Timespan::TimeDiff>(1000)*Poco::NumberParser::parse(it->second);
}
else if (it->first == "authMechanism")
{
authMechanism = it->second;
}
else if (it->first == "readPreference")
{
readPreference= it->second;
}
}
for (std::vector<std::string>::const_iterator it = strAddresses.cbegin();it != strAddresses.cend(); ++it)
{
newURI = *it;
theURI = Poco::URI(newURI);
std::string host = theURI.getHost();
Poco::UInt16 port = theURI.getPort();
if (port == 0) port = 27017;
connect(socketFactory.createSocket(host, port, connectTimeout, ssl));
_uri = newURI;
if (socketTimeout > 0)
{
_socket.setSendTimeout(socketTimeout);
_socket.setReceiveTimeout(socketTimeout);
}
if (strAddresses.size() > 1)
{
Poco::MongoDB::QueryRequest request("admin.$cmd");
request.setNumberToReturn(1);
request.selector().add("isMaster", 1);
Poco::MongoDB::ResponseMessage response;
sendRequest(request, response);
_uri = newURI;
if (!response.documents().empty())
{
Poco::MongoDB::Document::Ptr doc = response.documents()[0];
if (doc->get<bool>("ismaster") && readPreference == "primary")
{
break;
}
else if (!doc->get<bool>("ismaster") && readPreference == "secondary")
{
break;
}
else if (it + 1 == strAddresses.cend())
{
throw Poco::URISyntaxException(uri);
}
}
}
}
if (!userInfo.empty())
{
std::string username;
std::string password;
std::string::size_type pos = userInfo.find(':');
if (pos != std::string::npos)
{
username.assign(userInfo, 0, pos++);
password.assign(userInfo, pos, userInfo.size() - pos);
}
else username = userInfo;
Database database(databaseName);
if (!database.authenticate(*this, username, password, authMechanism))
throw Poco::NoPermissionException(Poco::format("Access to MongoDB database %s denied for user %s", databaseName, username));
}
}
void Connection::disconnect()
{
_socket.close();
}
void Connection::sendRequest(RequestMessage& request)
{
Poco::Net::SocketOutputStream sos(_socket);
request.send(sos);
}
void Connection::sendRequest(RequestMessage& request, ResponseMessage& response)
{
sendRequest(request);
Poco::Net::SocketInputStream sis(_socket);
response.read(sis);
}
void Connection::sendRequest(OpMsgMessage& request, OpMsgMessage& response)
{
Poco::Net::SocketOutputStream sos(_socket);
request.send(sos);
response.clear();
readResponse(response);
}
void Connection::sendRequest(OpMsgMessage& request)
{
request.setAcknowledgedRequest(false);
Poco::Net::SocketOutputStream sos(_socket);
request.send(sos);
}
void Connection::readResponse(OpMsgMessage& response)
{
Poco::Net::SocketInputStream sis(_socket);
response.read(sis);
}
} } // Poco::MongoDB

83
base/poco/MongoDB/src/Cursor.cpp
View File

@ -1,83 +0,0 @@
//
// Cursor.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: Cursor
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/Cursor.h"
#include "Poco/MongoDB/GetMoreRequest.h"
#include "Poco/MongoDB/KillCursorsRequest.h"
namespace Poco {
namespace MongoDB {
Cursor::Cursor(const std::string& db, const std::string& collection, QueryRequest::Flags flags):
_query(db + '.' + collection, flags)
{
}
Cursor::Cursor(const std::string& fullCollectionName, QueryRequest::Flags flags):
_query(fullCollectionName, flags)
{
}
Cursor::Cursor(const Document& aggregationResponse) :
_query(aggregationResponse.get<Poco::MongoDB::Document::Ptr>("cursor")->get<std::string>("ns")),
_response(aggregationResponse.get<Poco::MongoDB::Document::Ptr>("cursor")->get<Int64>("id"))
{
}
Cursor::~Cursor()
{
try
{
poco_assert_dbg(!_response.cursorID());
}
catch (...)
{
}
}
ResponseMessage& Cursor::next(Connection& connection)
{
if (_response.cursorID() == 0)
{
connection.sendRequest(_query, _response);
}
else
{
Poco::MongoDB::GetMoreRequest getMore(_query.fullCollectionName(), _response.cursorID());
getMore.setNumberToReturn(_query.getNumberToReturn());
_response.clear();
connection.sendRequest(getMore, _response);
}
return _response;
}
void Cursor::kill(Connection& connection)
{
if (_response.cursorID() != 0)
{
KillCursorsRequest killRequest;
killRequest.cursors().push_back(_response.cursorID());
connection.sendRequest(killRequest);
}
_response.clear();
}
} } // Namespace Poco::MongoDB

482
base/poco/MongoDB/src/Database.cpp
View File

@ -1,482 +0,0 @@
//
// Database.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: Database
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/Database.h"
#include "Poco/MongoDB/Binary.h"
#include "Poco/MD5Engine.h"
#include "Poco/SHA1Engine.h"
#include "Poco/PBKDF2Engine.h"
#include "Poco/HMACEngine.h"
#include "Poco/Base64Decoder.h"
#include "Poco/MemoryStream.h"
#include "Poco/StreamCopier.h"
#include "Poco/Exception.h"
#include "Poco/RandomStream.h"
#include "Poco/Random.h"
#include "Poco/Format.h"
#include "Poco/NumberParser.h"
#include <sstream>
#include <map>
namespace Poco {
namespace MongoDB {
const std::string Database::AUTH_MONGODB_CR("MONGODB-CR");
const std::string Database::AUTH_SCRAM_SHA1("SCRAM-SHA-1");
namespace
{
std::map<std::string, std::string> parseKeyValueList(const std::string& str)
{
std::map<std::string, std::string> kvm;
std::string::const_iterator it = str.begin();
std::string::const_iterator end = str.end();
while (it != end)
{
std::string k;
std::string v;
while (it != end && *it != '=') k += *it++;
if (it != end) ++it;
while (it != end && *it != ',') v += *it++;
if (it != end) ++it;
kvm[k] = v;
}
return kvm;
}
std::string decodeBase64(const std::string& base64)
{
Poco::MemoryInputStream istr(base64.data(), base64.size());
Poco::Base64Decoder decoder(istr);
std::string result;
Poco::StreamCopier::copyToString(decoder, result);
return result;
}
std::string encodeBase64(const std::string& data)
{
std::ostringstream ostr;
Poco::Base64Encoder encoder(ostr);
encoder.rdbuf()->setLineLength(0);
encoder << data;
encoder.close();
return ostr.str();
}
std::string digestToBinaryString(Poco::DigestEngine& engine)
{
Poco::DigestEngine::Digest d = engine.digest();
return std::string(reinterpret_cast<const char*>(&d[0]), d.size());
}
std::string digestToHexString(Poco::DigestEngine& engine)
{
Poco::DigestEngine::Digest d = engine.digest();
return Poco::DigestEngine::digestToHex(d);
}
std::string digestToBase64(Poco::DigestEngine& engine)
{
return encodeBase64(digestToBinaryString(engine));
}
std::string hashCredentials(const std::string& username, const std::string& password)
{
Poco::MD5Engine md5;
md5.update(username);
md5.update(std::string(":mongo:"));
md5.update(password);
return digestToHexString(md5);
}
std::string createNonce()
{
Poco::MD5Engine md5;
Poco::RandomInputStream randomStream;
Poco::Random random;
for (int i = 0; i < 4; i++)
{
md5.update(randomStream.get());
md5.update(random.nextChar());
}
return digestToHexString(md5);
}
}
Database::Database(const std::string& db):
_dbname(db)
{
}
Database::~Database()
{
}
bool Database::authenticate(Connection& connection, const std::string& username, const std::string& password, const std::string& method)
{
if (username.empty()) throw Poco::InvalidArgumentException("empty username");
if (password.empty()) throw Poco::InvalidArgumentException("empty password");
if (method == AUTH_MONGODB_CR)
return authCR(connection, username, password);
else if (method == AUTH_SCRAM_SHA1)
return authSCRAM(connection, username, password);
else
throw Poco::InvalidArgumentException("authentication method", method);
}
bool Database::authCR(Connection& connection, const std::string& username, const std::string& password)
{
std::string nonce;
Poco::SharedPtr<QueryRequest> pCommand = createCommand();
pCommand->selector().add<Poco::Int32>("getnonce", 1);
ResponseMessage response;
connection.sendRequest(*pCommand, response);
if (response.documents().size() > 0)
{
Document::Ptr pDoc = response.documents()[0];
if (pDoc->getInteger("ok") != 1) return false;
nonce = pDoc->get<std::string>("nonce", "");
if (nonce.empty()) throw Poco::ProtocolException("no nonce received");
}
else throw Poco::ProtocolException("empty response for getnonce");
std::string credsDigest = hashCredentials(username, password);
Poco::MD5Engine md5;
md5.update(nonce);
md5.update(username);
md5.update(credsDigest);
std::string key = digestToHexString(md5);
pCommand = createCommand();
pCommand->selector()
.add<Poco::Int32>("authenticate", 1)
.add<std::string>("user", username)
.add<std::string>("nonce", nonce)
.add<std::string>("key", key);
connection.sendRequest(*pCommand, response);
if (response.documents().size() > 0)
{
Document::Ptr pDoc = response.documents()[0];
return pDoc->getInteger("ok") == 1;
}
else throw Poco::ProtocolException("empty response for authenticate");
}
bool Database::authSCRAM(Connection& connection, const std::string& username, const std::string& password)
{
std::string clientNonce(createNonce());
std::string clientFirstMsg = Poco::format("n=%s,r=%s", username, clientNonce);
Poco::SharedPtr<QueryRequest> pCommand = createCommand();
pCommand->selector()
.add<Poco::Int32>("saslStart", 1)
.add<std::string>("mechanism", AUTH_SCRAM_SHA1)
.add<Binary::Ptr>("payload", new Binary(Poco::format("n,,%s", clientFirstMsg)));
ResponseMessage response;
connection.sendRequest(*pCommand, response);
Int32 conversationId = 0;
std::string serverFirstMsg;
if (response.documents().size() > 0)
{
Document::Ptr pDoc = response.documents()[0];
if (pDoc->getInteger("ok") == 1)
{
Binary::Ptr pPayload = pDoc->get<Binary::Ptr>("payload");
serverFirstMsg = pPayload->toRawString();
conversationId = pDoc->get<Int32>("conversationId");
}
else
{
if (pDoc->exists("errmsg"))
{
const Poco::MongoDB::Element::Ptr value = pDoc->get("errmsg");
auto message = static_cast<const Poco::MongoDB::ConcreteElement<std::string> &>(*value).value();
throw Poco::RuntimeException(message);
}
else
return false;
}
}
else throw Poco::ProtocolException("empty response for saslStart");
std::map<std::string, std::string> kvm = parseKeyValueList(serverFirstMsg);
const std::string serverNonce = kvm["r"];
const std::string salt = decodeBase64(kvm["s"]);
const unsigned iterations = Poco::NumberParser::parseUnsigned(kvm["i"]);
const Poco::UInt32 dkLen = 20;
std::string hashedPassword = hashCredentials(username, password);
Poco::PBKDF2Engine<Poco::HMACEngine<Poco::SHA1Engine> > pbkdf2(salt, iterations, dkLen);
pbkdf2.update(hashedPassword);
std::string saltedPassword = digestToBinaryString(pbkdf2);
std::string clientFinalNoProof = Poco::format("c=biws,r=%s", serverNonce);
std::string authMessage = Poco::format("%s,%s,%s", clientFirstMsg, serverFirstMsg, clientFinalNoProof);
Poco::HMACEngine<Poco::SHA1Engine> hmacKey(saltedPassword);
hmacKey.update(std::string("Client Key"));
std::string clientKey = digestToBinaryString(hmacKey);
Poco::SHA1Engine sha1;
sha1.update(clientKey);
std::string storedKey = digestToBinaryString(sha1);
Poco::HMACEngine<Poco::SHA1Engine> hmacSig(storedKey);
hmacSig.update(authMessage);
std::string clientSignature = digestToBinaryString(hmacSig);
std::string clientProof(clientKey);
for (std::size_t i = 0; i < clientProof.size(); i++)
{
clientProof[i] ^= clientSignature[i];
}
std::string clientFinal = Poco::format("%s,p=%s", clientFinalNoProof, encodeBase64(clientProof));
pCommand = createCommand();
pCommand->selector()
.add<Poco::Int32>("saslContinue", 1)
.add<Poco::Int32>("conversationId", conversationId)
.add<Binary::Ptr>("payload", new Binary(clientFinal));
std::string serverSecondMsg;
connection.sendRequest(*pCommand, response);
if (response.documents().size() > 0)
{
Document::Ptr pDoc = response.documents()[0];
if (pDoc->getInteger("ok") == 1)
{
Binary::Ptr pPayload = pDoc->get<Binary::Ptr>("payload");
serverSecondMsg = pPayload->toRawString();
}
else
{
if (pDoc->exists("errmsg"))
{
const Poco::MongoDB::Element::Ptr value = pDoc->get("errmsg");
auto message = static_cast<const Poco::MongoDB::ConcreteElement<std::string> &>(*value).value();
throw Poco::RuntimeException(message);
}
else
return false;
}
}
else throw Poco::ProtocolException("empty response for saslContinue");
Poco::HMACEngine<Poco::SHA1Engine> hmacSKey(saltedPassword);
hmacSKey.update(std::string("Server Key"));
std::string serverKey = digestToBinaryString(hmacSKey);
Poco::HMACEngine<Poco::SHA1Engine> hmacSSig(serverKey);
hmacSSig.update(authMessage);
std::string serverSignature = digestToBase64(hmacSSig);
kvm = parseKeyValueList(serverSecondMsg);
std::string serverSignatureReceived = kvm["v"];
if (serverSignature != serverSignatureReceived)
throw Poco::ProtocolException("server signature verification failed");
pCommand = createCommand();
pCommand->selector()
.add<Poco::Int32>("saslContinue", 1)
.add<Poco::Int32>("conversationId", conversationId)
.add<Binary::Ptr>("payload", new Binary);
connection.sendRequest(*pCommand, response);
if (response.documents().size() > 0)
{
Document::Ptr pDoc = response.documents()[0];
if (pDoc->getInteger("ok") == 1)
{
return true;
}
else
{
if (pDoc->exists("errmsg"))
{
const Poco::MongoDB::Element::Ptr value = pDoc->get("errmsg");
auto message = static_cast<const Poco::MongoDB::ConcreteElement<std::string> &>(*value).value();
throw Poco::RuntimeException(message);
}
else
return false;
}
}
else throw Poco::ProtocolException("empty response for saslContinue");
}
Document::Ptr Database::queryBuildInfo(Connection& connection) const
{
// build info can be issued on "config" system database
Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
request->selector().add("buildInfo", 1);
Poco::MongoDB::ResponseMessage response;
connection.sendRequest(*request, response);
Document::Ptr buildInfo;
if ( response.documents().size() > 0 )
{
buildInfo = response.documents()[0];
}
else
{
throw Poco::ProtocolException("Didn't get a response from the buildinfo command");
}
return buildInfo;
}
Document::Ptr Database::queryServerHello(Connection& connection, bool old) const
{
// hello can be issued on "config" system database
Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
// 'hello' command was previously called 'isMaster'
std::string command_name;
if (old)
command_name = "isMaster";
else
command_name = "hello";
request->selector().add(command_name, 1);
Poco::MongoDB::ResponseMessage response;
connection.sendRequest(*request, response);
Document::Ptr hello;
if ( response.documents().size() > 0 )
{
hello = response.documents()[0];
}
else
{
throw Poco::ProtocolException("Didn't get a response from the hello command");
}
return hello;
}
Int64 Database::count(Connection& connection, const std::string& collectionName) const
{
Poco::SharedPtr<Poco::MongoDB::QueryRequest> countRequest = createCountRequest(collectionName);
Poco::MongoDB::ResponseMessage response;
connection.sendRequest(*countRequest, response);
if (response.documents().size() > 0)
{
Poco::MongoDB::Document::Ptr doc = response.documents()[0];
return doc->getInteger("n");
}
return -1;
}
Poco::MongoDB::Document::Ptr Database::ensureIndex(Connection& connection, const std::string& collection, const std::string& indexName, Poco::MongoDB::Document::Ptr keys, bool unique, bool background, int version, int ttl)
{
Poco::MongoDB::Document::Ptr index = new Poco::MongoDB::Document();
index->add("ns", _dbname + "." + collection);
index->add("name", indexName);
index->add("key", keys);
if (version > 0)
{
index->add("version", version);
}
if (unique)
{
index->add("unique", true);
}
if (background)
{
index->add("background", true);
}
if (ttl > 0)
{
index->add("expireAfterSeconds", ttl);
}
Poco::SharedPtr<Poco::MongoDB::InsertRequest> insertRequest = createInsertRequest("system.indexes");
insertRequest->documents().push_back(index);
connection.sendRequest(*insertRequest);
return getLastErrorDoc(connection);
}
Document::Ptr Database::getLastErrorDoc(Connection& connection) const
{
Document::Ptr errorDoc;
Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
request->setNumberToReturn(1);
request->selector().add("getLastError", 1);
Poco::MongoDB::ResponseMessage response;
connection.sendRequest(*request, response);
if (response.documents().size() > 0)
{
errorDoc = response.documents()[0];
}
return errorDoc;
}
std::string Database::getLastError(Connection& connection) const
{
Document::Ptr errorDoc = getLastErrorDoc(connection);
if (!errorDoc.isNull() && errorDoc->isType<std::string>("err"))
{
return errorDoc->get<std::string>("err");
}
return "";
}
Poco::SharedPtr<Poco::MongoDB::QueryRequest> Database::createCountRequest(const std::string& collectionName) const
{
Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
request->setNumberToReturn(1);
request->selector().add("count", collectionName);
return request;
}
} } // namespace Poco::MongoDB

54
base/poco/MongoDB/src/DeleteRequest.cpp
View File

@ -1,54 +0,0 @@
//
// DeleteRequest.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: DeleteRequest
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/DeleteRequest.h"
namespace Poco {
namespace MongoDB {
DeleteRequest::DeleteRequest(const std::string& collectionName, DeleteRequest::Flags flags):
RequestMessage(MessageHeader::OP_DELETE),
_flags(flags),
_fullCollectionName(collectionName),
_selector()
{
}
DeleteRequest::DeleteRequest(const std::string& collectionName, bool justOne):
RequestMessage(MessageHeader::OP_DELETE),
_flags(justOne ? DELETE_SINGLE_REMOVE : DELETE_DEFAULT),
_fullCollectionName(collectionName),
_selector()
{
}
DeleteRequest::~DeleteRequest()
{
}
void DeleteRequest::buildRequest(BinaryWriter& writer)
{
writer << 0; // 0 - reserved for future use
BSONWriter(writer).writeCString(_fullCollectionName);
writer << _flags;
_selector.write(writer);
}
} } // namespace Poco::MongoDB

227
base/poco/MongoDB/src/Document.cpp
View File

@ -1,227 +0,0 @@
//
// Document.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: Document
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/Document.h"
#include "Poco/MongoDB/Binary.h"
#include "Poco/MongoDB/ObjectId.h"
#include "Poco/MongoDB/Array.h"
#include "Poco/MongoDB/RegularExpression.h"
#include "Poco/MongoDB/JavaScriptCode.h"
#include <sstream>
namespace Poco {
namespace MongoDB {
Document::Document()
{
}
Document::~Document()
{
}
Array& Document::addNewArray(const std::string& name)
{
Array::Ptr newArray = new Array();
add(name, newArray);
return *newArray;
}
Element::Ptr Document::get(const std::string& name) const
{
Element::Ptr element;
ElementSet::const_iterator it = std::find_if(_elements.begin(), _elements.end(), ElementFindByName(name));
if (it != _elements.end())
{
return *it;
}
return element;
}
Int64 Document::getInteger(const std::string& name) const
{
Element::Ptr element = get(name);
if (element.isNull()) throw Poco::NotFoundException(name);
if (ElementTraits<double>::TypeId == element->type())
{
ConcreteElement<double>* concrete = dynamic_cast<ConcreteElement<double>*>(element.get());
if (concrete) return static_cast<Int64>(concrete->value());
}
else if (ElementTraits<Int32>::TypeId == element->type())
{
ConcreteElement<Int32>* concrete = dynamic_cast<ConcreteElement<Int32>*>(element.get());
if (concrete) return concrete->value();
}
else if (ElementTraits<Int64>::TypeId == element->type())
{
ConcreteElement<Int64>* concrete = dynamic_cast<ConcreteElement<Int64>*>(element.get());
if (concrete) return concrete->value();
}
throw Poco::BadCastException("Invalid type mismatch!");
}
void Document::read(BinaryReader& reader)
{
int size;
reader >> size;
unsigned char type;
reader >> type;
while (type != '\0')
{
Element::Ptr element;
std::string name = BSONReader(reader).readCString();
switch (type)
{
case ElementTraits<double>::TypeId:
element = new ConcreteElement<double>(name, 0);
break;
case ElementTraits<Int32>::TypeId:
element = new ConcreteElement<Int32>(name, 0);
break;
case ElementTraits<std::string>::TypeId:
element = new ConcreteElement<std::string>(name, "");
break;
case ElementTraits<Document::Ptr>::TypeId:
element = new ConcreteElement<Document::Ptr>(name, new Document);
break;
case ElementTraits<Array::Ptr>::TypeId:
element = new ConcreteElement<Array::Ptr>(name, new Array);
break;
case ElementTraits<Binary::Ptr>::TypeId:
element = new ConcreteElement<Binary::Ptr>(name, new Binary);
break;
case ElementTraits<ObjectId::Ptr>::TypeId:
element = new ConcreteElement<ObjectId::Ptr>(name, new ObjectId);
break;
case ElementTraits<bool>::TypeId:
element = new ConcreteElement<bool>(name, false);
break;
case ElementTraits<Poco::Timestamp>::TypeId:
element = new ConcreteElement<Poco::Timestamp>(name, Poco::Timestamp());
break;
case ElementTraits<BSONTimestamp>::TypeId:
element = new ConcreteElement<BSONTimestamp>(name, BSONTimestamp());
break;
case ElementTraits<NullValue>::TypeId:
element = new ConcreteElement<NullValue>(name, NullValue(0));
break;
case ElementTraits<RegularExpression::Ptr>::TypeId:
element = new ConcreteElement<RegularExpression::Ptr>(name, new RegularExpression());
break;
case ElementTraits<JavaScriptCode::Ptr>::TypeId:
element = new ConcreteElement<JavaScriptCode::Ptr>(name, new JavaScriptCode());
break;
case ElementTraits<Int64>::TypeId:
element = new ConcreteElement<Int64>(name, 0);
break;
default:
{
std::stringstream ss;
ss << "Element " << name << " contains an unsupported type 0x" << std::hex << (int) type;
throw Poco::NotImplementedException(ss.str());
}
//TODO: x0F -> JavaScript code with scope
// xFF -> Min Key
// x7F -> Max Key
}
element->read(reader);
_elements.push_back(element);
reader >> type;
}
}
std::string Document::toString(int indent) const
{
std::ostringstream oss;
oss << '{';
if (indent > 0) oss << std::endl;
for (ElementSet::const_iterator it = _elements.begin(); it != _elements.end(); ++it)
{
if (it != _elements.begin())
{
oss << ',';
if (indent > 0) oss << std::endl;
}
for (int i = 0; i < indent; ++i) oss << ' ';
oss << '"' << (*it)->name() << '"';
oss << (indent > 0 ? " : " : ":");
oss << (*it)->toString(indent > 0 ? indent + 2 : 0);
}
if (indent > 0)
{
oss << std::endl;
if (indent >= 2) indent -= 2;
for (int i = 0; i < indent; ++i) oss << ' ';
}
oss << '}';
return oss.str();
}
void Document::write(BinaryWriter& writer)
{
if (_elements.empty())
{
writer << 5;
}
else
{
std::stringstream sstream;
Poco::BinaryWriter tempWriter(sstream, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
for (ElementSet::iterator it = _elements.begin(); it != _elements.end(); ++it)
{
tempWriter << static_cast<unsigned char>((*it)->type());
BSONWriter(tempWriter).writeCString((*it)->name());
Element::Ptr element = *it;
element->write(tempWriter);
}
tempWriter.flush();
Poco::Int32 len = static_cast<Poco::Int32>(5 + sstream.tellp()); /* 5 = sizeof(len) + 0-byte */
writer << len;
writer.writeRaw(sstream.str());
}
writer << '\0';
}
} } // namespace Poco::MongoDB

32
base/poco/MongoDB/src/Element.cpp
View File

@ -1,32 +0,0 @@
//
// Element.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: Element
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/Element.h"
namespace Poco {
namespace MongoDB {
Element::Element(const std::string& name) : _name(name)
{
}
Element::~Element()
{
}
} } // namespace Poco::MongoDB

46
base/poco/MongoDB/src/GetMoreRequest.cpp
View File

@ -1,46 +0,0 @@
//
// GetMoreRequest.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: GetMoreRequest
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/GetMoreRequest.h"
#include "Poco/MongoDB/Element.h"
namespace Poco {
namespace MongoDB {
GetMoreRequest::GetMoreRequest(const std::string& collectionName, Int64 cursorID):
RequestMessage(MessageHeader::OP_GET_MORE),
_fullCollectionName(collectionName),
_numberToReturn(100),
_cursorID(cursorID)
{
}
GetMoreRequest::~GetMoreRequest()
{
}
void GetMoreRequest::buildRequest(BinaryWriter& writer)
{
writer << 0; // 0 - reserved for future use
BSONWriter(writer).writeCString(_fullCollectionName);
writer << _numberToReturn;
writer << _cursorID;
}
} } // namespace Poco::MongoDB

49
base/poco/MongoDB/src/InsertRequest.cpp
View File

@ -1,49 +0,0 @@
//
// InsertRequest.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: InsertRequest
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/InsertRequest.h"
namespace Poco {
namespace MongoDB {
InsertRequest::InsertRequest(const std::string& collectionName, Flags flags):
RequestMessage(MessageHeader::OP_INSERT),
_flags(flags),
_fullCollectionName(collectionName)
{
}
InsertRequest::~InsertRequest()
{
}
void InsertRequest::buildRequest(BinaryWriter& writer)
{
poco_assert (!_documents.empty());
writer << _flags;
BSONWriter bsonWriter(writer);
bsonWriter.writeCString(_fullCollectionName);
for (Document::Vector::iterator it = _documents.begin(); it != _documents.end(); ++it)
{
bsonWriter.write(*it);
}
}
} } // namespace Poco::MongoDB

33
base/poco/MongoDB/src/JavaScriptCode.cpp
View File

@ -1,33 +0,0 @@
//
// JavaScriptCode.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: JavaScriptCode
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/JavaScriptCode.h"
namespace Poco {
namespace MongoDB {
JavaScriptCode::JavaScriptCode()
{
}
JavaScriptCode::~JavaScriptCode()
{
}
} } // namespace Poco::MongoDB

44
base/poco/MongoDB/src/KillCursorsRequest.cpp
View File

@ -1,44 +0,0 @@
//
// KillCursorsRequest.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: KillCursorsRequest
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/KillCursorsRequest.h"
namespace Poco {
namespace MongoDB {
KillCursorsRequest::KillCursorsRequest():
RequestMessage(MessageHeader::OP_KILL_CURSORS)
{
}
KillCursorsRequest::~KillCursorsRequest()
{
}
void KillCursorsRequest::buildRequest(BinaryWriter& writer)
{
writer << 0; // 0 - reserved for future use
writer << static_cast<Poco::UInt64>(_cursors.size());
for (std::vector<Int64>::iterator it = _cursors.begin(); it != _cursors.end(); ++it)
{
writer << *it;
}
}
} } // namespace Poco::MongoDB

33
base/poco/MongoDB/src/Message.cpp
View File

@ -1,33 +0,0 @@
//
// Message.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: Message
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/Message.h"
namespace Poco {
namespace MongoDB {
Message::Message(MessageHeader::OpCode opcode):
_header(opcode)
{
}
Message::~Message()
{
}
} } // namespace Poco::MongoDB

63
base/poco/MongoDB/src/MessageHeader.cpp
View File

@ -1,63 +0,0 @@
//
// MessageHeader.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: MessageHeader
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/Message.h"
#include "Poco/Exception.h"
namespace Poco {
namespace MongoDB {
MessageHeader::MessageHeader(OpCode opCode):
_messageLength(0),
_requestID(0),
_responseTo(0),
_opCode(opCode)
{
}
MessageHeader::~MessageHeader()
{
}
void MessageHeader::read(BinaryReader& reader)
{
reader >> _messageLength;
reader >> _requestID;
reader >> _responseTo;
Int32 opCode;
reader >> opCode;
_opCode = static_cast<OpCode>(opCode);
if (!reader.good())
{
throw IOException("Failed to read from socket");
}
}
void MessageHeader::write(BinaryWriter& writer)
{
writer << _messageLength;
writer << _requestID;
writer << _responseTo;
writer << static_cast<Int32>(_opCode);
}
} } // namespace Poco::MongoDB

66
base/poco/MongoDB/src/ObjectId.cpp
View File

@ -1,66 +0,0 @@
//
// ObjectId.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: ObjectId
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/ObjectId.h"
#include "Poco/Format.h"
#include <cstring>
namespace Poco {
namespace MongoDB {
ObjectId::ObjectId()
{
std::memset(_id, 0, sizeof(_id));
}
ObjectId::ObjectId(const std::string& id)
{
poco_assert_dbg(id.size() == 24);
const char* p = id.c_str();
for (std::size_t i = 0; i < 12; ++i)
{
_id[i] = fromHex(p);
p += 2;
}
}
ObjectId::ObjectId(const ObjectId& copy)
{
std::memcpy(_id, copy._id, sizeof(_id));
}
ObjectId::~ObjectId()
{
}
std::string ObjectId::toString(const std::string& fmt) const
{
std::string s;
for (int i = 0; i < 12; ++i)
{
s += Poco::format(fmt, (unsigned int) _id[i]);
}
return s;
}
} } // namespace Poco::MongoDB

187
base/poco/MongoDB/src/OpMsgCursor.cpp
View File

@ -1,187 +0,0 @@
//
// OpMsgCursor.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: OpMsgCursor
//
// Copyright (c) 2022, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/OpMsgCursor.h"
#include "Poco/MongoDB/Array.h"
//
// NOTE:
//
// MongoDB specification indicates that the flag MSG_EXHAUST_ALLOWED shall be
// used in the request when the receiver is ready to receive multiple messages
// without sending additional requests in between. Sender (MongoDB) indicates
// that more messages follow with flag MSG_MORE_TO_COME.
//
// It seems that this does not work properly. MSG_MORE_TO_COME is set and reading
// next messages sometimes works, however often the data is missing in response
// or the message header contains wrong message length and reading blocks.
// Opcode in the header is correct.
//
// Using MSG_EXHAUST_ALLOWED is therefore currently disabled.
//
// It seems that related JIRA ticket is:
//
// https://jira.mongodb.org/browse/SERVER-57297
//
// https://github.com/mongodb/specifications/blob/master/source/message/OP_MSG.rst
//
#define MONGODB_EXHAUST_ALLOWED_WORKS false
namespace Poco {
namespace MongoDB {
[[ maybe_unused ]] static const std::string keyCursor {"cursor"};
[[ maybe_unused ]] static const std::string keyFirstBatch {"firstBatch"};
[[ maybe_unused ]] static const std::string keyNextBatch {"nextBatch"};
static Poco::Int64 cursorIdFromResponse(const MongoDB::Document& doc);
OpMsgCursor::OpMsgCursor(const std::string& db, const std::string& collection):
#if MONGODB_EXHAUST_ALLOWED_WORKS
_query(db, collection, OpMsgMessage::MSG_EXHAUST_ALLOWED)
#else
_query(db, collection)
#endif
{
}
OpMsgCursor::~OpMsgCursor()
{
try
{
poco_assert_dbg(_cursorID == 0);
}
catch (...)
{
}
}
void OpMsgCursor::setEmptyFirstBatch(bool empty)
{
_emptyFirstBatch = empty;
}
bool OpMsgCursor::emptyFirstBatch() const
{
return _emptyFirstBatch;
}
void OpMsgCursor::setBatchSize(Int32 batchSize)
{
_batchSize = batchSize;
}
Int32 OpMsgCursor::batchSize() const
{
return _batchSize;
}
OpMsgMessage& OpMsgCursor::next(Connection& connection)
{
if (_cursorID == 0)
{
_response.clear();
if (_emptyFirstBatch || _batchSize > 0)
{
Int32 bsize = _emptyFirstBatch ? 0 : _batchSize;
if (_query.commandName() == OpMsgMessage::CMD_FIND)
{
_query.body().add("batchSize", bsize);
}
else if (_query.commandName() == OpMsgMessage::CMD_AGGREGATE)
{
auto& cursorDoc = _query.body().addNewDocument("cursor");
cursorDoc.add("batchSize", bsize);
}
}
connection.sendRequest(_query, _response);
const auto& rdoc = _response.body();
_cursorID = cursorIdFromResponse(rdoc);
}
else
{
#if MONGODB_EXHAUST_ALLOWED_WORKS
std::cout << "Response flags: " << _response.flags() << std::endl;
if (_response.flags() & OpMsgMessage::MSG_MORE_TO_COME)
{
std::cout << "More to come. Reading more response: " << std::endl;
_response.clear();
connection.readResponse(_response);
}
else
#endif
{
_response.clear();
_query.setCursor(_cursorID, _batchSize);
connection.sendRequest(_query, _response);
}
}
const auto& rdoc = _response.body();
_cursorID = cursorIdFromResponse(rdoc);
return _response;
}
void OpMsgCursor::kill(Connection& connection)
{
_response.clear();
if (_cursorID != 0)
{
_query.setCommandName(OpMsgMessage::CMD_KILL_CURSORS);
MongoDB::Array::Ptr cursors = new MongoDB::Array();
cursors->add<Poco::Int64>(_cursorID);
_query.body().add("cursors", cursors);
connection.sendRequest(_query, _response);
const auto killed = _response.body().get<MongoDB::Array::Ptr>("cursorsKilled", nullptr);
if (!killed || killed->size() != 1 || killed->get<Poco::Int64>(0, -1) != _cursorID)
{
throw Poco::ProtocolException("Cursor not killed as expected: " + std::to_string(_cursorID));
}
_cursorID = 0;
_query.clear();
_response.clear();
}
}
Poco::Int64 cursorIdFromResponse(const MongoDB::Document& doc)
{
Poco::Int64 id {0};
auto cursorDoc = doc.get<Document::Ptr>(keyCursor, nullptr);
if(cursorDoc)
{
id = cursorDoc->get<Poco::Int64>("id", 0);
}
return id;
}
} } // Namespace Poco::MongoDB

412
base/poco/MongoDB/src/OpMsgMessage.cpp
View File

@ -1,412 +0,0 @@
//
// OpMsgMessage.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: OpMsgMessage
//
// Copyright (c) 2022, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/OpMsgMessage.h"
#include "Poco/MongoDB/MessageHeader.h"
#include "Poco/MongoDB/Array.h"
#include "Poco/StreamCopier.h"
#include "Poco/Logger.h"
#define POCO_MONGODB_DUMP false
namespace Poco {
namespace MongoDB {
// Query and write
const std::string OpMsgMessage::CMD_INSERT { "insert" };
const std::string OpMsgMessage::CMD_DELETE { "delete" };
const std::string OpMsgMessage::CMD_UPDATE { "update" };
const std::string OpMsgMessage::CMD_FIND { "find" };
const std::string OpMsgMessage::CMD_FIND_AND_MODIFY { "findAndModify" };
const std::string OpMsgMessage::CMD_GET_MORE { "getMore" };
// Aggregation
const std::string OpMsgMessage::CMD_AGGREGATE { "aggregate" };
const std::string OpMsgMessage::CMD_COUNT { "count" };
const std::string OpMsgMessage::CMD_DISTINCT { "distinct" };
const std::string OpMsgMessage::CMD_MAP_REDUCE { "mapReduce" };
// Replication and administration
const std::string OpMsgMessage::CMD_HELLO { "hello" };
const std::string OpMsgMessage::CMD_REPL_SET_GET_STATUS { "replSetGetStatus" };
const std::string OpMsgMessage::CMD_REPL_SET_GET_CONFIG { "replSetGetConfig" };
const std::string OpMsgMessage::CMD_CREATE { "create" };
const std::string OpMsgMessage::CMD_CREATE_INDEXES { "createIndexes" };
const std::string OpMsgMessage::CMD_DROP { "drop" };
const std::string OpMsgMessage::CMD_DROP_DATABASE { "dropDatabase" };
const std::string OpMsgMessage::CMD_KILL_CURSORS { "killCursors" };
const std::string OpMsgMessage::CMD_LIST_DATABASES { "listDatabases" };
const std::string OpMsgMessage::CMD_LIST_INDEXES { "listIndexes" };
// Diagnostic
const std::string OpMsgMessage::CMD_BUILD_INFO { "buildInfo" };
const std::string OpMsgMessage::CMD_COLL_STATS { "collStats" };
const std::string OpMsgMessage::CMD_DB_STATS { "dbStats" };
const std::string OpMsgMessage::CMD_HOST_INFO { "hostInfo" };
static const std::string& commandIdentifier(const std::string& command);
/// Commands have different names for the payload that is sent in a separate section
static const std::string keyCursor {"cursor"};
static const std::string keyFirstBatch {"firstBatch"};
static const std::string keyNextBatch {"nextBatch"};
OpMsgMessage::OpMsgMessage() :
Message(MessageHeader::OP_MSG)
{
}
OpMsgMessage::OpMsgMessage(const std::string& databaseName, const std::string& collectionName, UInt32 flags) :
Message(MessageHeader::OP_MSG),
_databaseName(databaseName),
_collectionName(collectionName),
_flags(flags)
{
}
OpMsgMessage::~OpMsgMessage()
{
}
const std::string& OpMsgMessage::databaseName() const
{
return _databaseName;
}
const std::string& OpMsgMessage::collectionName() const
{
return _collectionName;
}
void OpMsgMessage::setCommandName(const std::string& command)
{
_commandName = command;
_body.clear();
// IMPORTANT: Command name must be first
if (_collectionName.empty())
{
// Collection is not specified. It is assumed that this particular command does
// not need it.
_body.add(_commandName, Int32(1));
}
else
{
_body.add(_commandName, _collectionName);
}
_body.add("$db", _databaseName);
}
void OpMsgMessage::setCursor(Poco::Int64 cursorID, Poco::Int32 batchSize)
{
_commandName = OpMsgMessage::CMD_GET_MORE;
_body.clear();
// IMPORTANT: Command name must be first
_body.add(_commandName, cursorID);
_body.add("$db", _databaseName);
_body.add("collection", _collectionName);
if (batchSize > 0)
{
_body.add("batchSize", batchSize);
}
}
const std::string& OpMsgMessage::commandName() const
{
return _commandName;
}
void OpMsgMessage::setAcknowledgedRequest(bool ack)
{
const auto& id = commandIdentifier(_commandName);
if (id.empty())
return;
_acknowledged = ack;
auto writeConcern = _body.get<Document::Ptr>("writeConcern", nullptr);
if (writeConcern)
writeConcern->remove("w");
if (ack)
{
_flags = _flags & (~MSG_MORE_TO_COME);
}
else
{
_flags = _flags | MSG_MORE_TO_COME;
if (!writeConcern)
_body.addNewDocument("writeConcern").add("w", 0);
else
writeConcern->add("w", 0);
}
}
bool OpMsgMessage::acknowledgedRequest() const
{
return _acknowledged;
}
UInt32 OpMsgMessage::flags() const
{
return _flags;
}
Document& OpMsgMessage::body()
{
return _body;
}
const Document& OpMsgMessage::body() const
{
return _body;
}
Document::Vector& OpMsgMessage::documents()
{
return _documents;
}
const Document::Vector& OpMsgMessage::documents() const
{
return _documents;
}
bool OpMsgMessage::responseOk() const
{
Poco::Int64 ok {false};
if (_body.exists("ok"))
{
ok = _body.getInteger("ok");
}
return (ok != 0);
}
void OpMsgMessage::clear()
{
_flags = MSG_FLAGS_DEFAULT;
_commandName.clear();
_body.clear();
_documents.clear();
}
void OpMsgMessage::send(std::ostream& ostr)
{
BinaryWriter socketWriter(ostr, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
// Serialise the body
std::stringstream ss;
BinaryWriter writer(ss, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
writer << _flags;
writer << PAYLOAD_TYPE_0;
_body.write(writer);
if (!_documents.empty())
{
// Serialise attached documents
std::stringstream ssdoc;
BinaryWriter wdoc(ssdoc, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
for (auto& doc: _documents)
{
doc->write(wdoc);
}
wdoc.flush();
const std::string& identifier = commandIdentifier(_commandName);
const Poco::Int32 size = static_cast<Poco::Int32>(sizeof(size) + identifier.size() + 1 + ssdoc.tellp());
writer << PAYLOAD_TYPE_1;
writer << size;
writer.writeCString(identifier.c_str());
StreamCopier::copyStream(ssdoc, ss);
}
writer.flush();
#if POCO_MONGODB_DUMP
const std::string section = ss.str();
std::string dump;
Logger::formatDump(dump, section.data(), section.length());
std::cout << dump << std::endl;
#endif
messageLength(static_cast<Poco::Int32>(ss.tellp()));
_header.write(socketWriter);
StreamCopier::copyStream(ss, ostr);
ostr.flush();
}
void OpMsgMessage::read(std::istream& istr)
{
std::string message;
{
BinaryReader reader(istr, BinaryReader::LITTLE_ENDIAN_BYTE_ORDER);
_header.read(reader);
poco_assert_dbg(_header.opCode() == _header.OP_MSG);
const std::streamsize remainingSize {_header.getMessageLength() - _header.MSG_HEADER_SIZE };
message.reserve(remainingSize);
#if POCO_MONGODB_DUMP
std::cout
<< "Message hdr: " << _header.getMessageLength() << " " << remainingSize << " "
<< _header.opCode() << " " << _header.getRequestID() << " " << _header.responseTo()
<< std::endl;
#endif
reader.readRaw(remainingSize, message);
#if POCO_MONGODB_DUMP
std::string dump;
Logger::formatDump(dump, message.data(), message.length());
std::cout << dump << std::endl;
#endif
}
// Read complete message and then interpret it.
std::istringstream msgss(message);
BinaryReader reader(msgss, BinaryReader::LITTLE_ENDIAN_BYTE_ORDER);
Poco::UInt8 payloadType {0xFF};
reader >> _flags;
reader >> payloadType;
poco_assert_dbg(payloadType == PAYLOAD_TYPE_0);
_body.read(reader);
// Read next sections from the buffer
while (msgss.good())
{
// NOTE: Not tested yet with database, because it returns everything in the body.
// Does MongoDB ever return documents as Payload type 1?
reader >> payloadType;
if (!msgss.good())
{
break;
}
poco_assert_dbg(payloadType == PAYLOAD_TYPE_1);
#if POCO_MONGODB_DUMP
std::cout << "section payload: " << payloadType << std::endl;
#endif
Poco::Int32 sectionSize {0};
reader >> sectionSize;
poco_assert_dbg(sectionSize > 0);
#if POCO_MONGODB_DUMP
std::cout << "section size: " << sectionSize << std::endl;
#endif
std::streamoff offset = sectionSize - sizeof(sectionSize);
std::streampos endOfSection = msgss.tellg() + offset;
std::string identifier;
reader.readCString(identifier);
#if POCO_MONGODB_DUMP
std::cout << "section identifier: " << identifier << std::endl;
#endif
// Loop to read documents from this section.
while (msgss.tellg() < endOfSection)
{
#if POCO_MONGODB_DUMP
std::cout << "section doc: " << msgss.tellg() << " " << endOfSection << std::endl;
#endif
Document::Ptr doc = new Document();
doc->read(reader);
_documents.push_back(doc);
if (msgss.tellg() < 0)
{
break;
}
}
}
// Extract documents from the cursor batch if they are there.
MongoDB::Array::Ptr batch;
auto curDoc = _body.get<MongoDB::Document::Ptr>(keyCursor, nullptr);
if (curDoc)
{
batch = curDoc->get<MongoDB::Array::Ptr>(keyFirstBatch, nullptr);
if (!batch)
{
batch = curDoc->get<MongoDB::Array::Ptr>(keyNextBatch, nullptr);
}
}
if (batch)
{
for(std::size_t i = 0; i < batch->size(); i++)
{
const auto& d = batch->get<MongoDB::Document::Ptr>(i, nullptr);
if (d)
{
_documents.push_back(d);
}
}
}
}
const std::string& commandIdentifier(const std::string& command)
{
// Names of identifiers for commands that send bulk documents in the request
// The identifier is set in the section type 1.
static std::map<std::string, std::string> identifiers {
{ OpMsgMessage::CMD_INSERT, "documents" },
{ OpMsgMessage::CMD_DELETE, "deletes" },
{ OpMsgMessage::CMD_UPDATE, "updates" },
// Not sure if create index can send document section
{ OpMsgMessage::CMD_CREATE_INDEXES, "indexes" }
};
const auto i = identifiers.find(command);
if (i != identifiers.end())
{
return i->second;
}
// This likely means that documents are incorrectly set for a command
// that does not send list of documents in section type 1.
static const std::string emptyIdentifier;
return emptyIdentifier;
}
} } // namespace Poco::MongoDB

54
base/poco/MongoDB/src/QueryRequest.cpp
View File

@ -1,54 +0,0 @@
//
// QueryRequest.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: QueryRequest
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/QueryRequest.h"
namespace Poco {
namespace MongoDB {
QueryRequest::QueryRequest(const std::string& collectionName, QueryRequest::Flags flags):
RequestMessage(MessageHeader::OP_QUERY),
_flags(flags),
_fullCollectionName(collectionName),
_numberToSkip(0),
_numberToReturn(100),
_selector(),
_returnFieldSelector()
{
}
QueryRequest::~QueryRequest()
{
}
void QueryRequest::buildRequest(BinaryWriter& writer)
{
writer << _flags;
BSONWriter(writer).writeCString(_fullCollectionName);
writer << _numberToSkip;
writer << _numberToReturn;
_selector.write(writer);
if (!_returnFieldSelector.empty())
{
_returnFieldSelector.write(writer);
}
}
} } // namespace Poco::MongoDB

71
base/poco/MongoDB/src/RegularExpression.cpp
View File

@ -1,71 +0,0 @@
//
// RegularExpression.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: RegularExpression
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/RegularExpression.h"
#include <sstream>
namespace Poco {
namespace MongoDB {
RegularExpression::RegularExpression()
{
}
RegularExpression::RegularExpression(const std::string& pattern, const std::string& options):
_pattern(pattern),
_options(options)
{
}
RegularExpression::~RegularExpression()
{
}
SharedPtr<Poco::RegularExpression> RegularExpression::createRE() const
{
int options = 0;
for (std::string::const_iterator optIt = _options.begin(); optIt != _options.end(); ++optIt)
{
switch (*optIt)
{
case 'i': // Case Insensitive
options |= Poco::RegularExpression::RE_CASELESS;
break;
case 'm': // Multiline matching
options |= Poco::RegularExpression::RE_MULTILINE;
break;
case 'x': // Verbose mode
//No equivalent in Poco
break;
case 'l': // \w \W Locale dependent
//No equivalent in Poco
break;
case 's': // Dotall mode
options |= Poco::RegularExpression::RE_DOTALL;
break;
case 'u': // \w \W Unicode
//No equivalent in Poco
break;
}
}
return new Poco::RegularExpression(_pattern, options);
}
} } // namespace Poco::MongoDB

89
base/poco/MongoDB/src/ReplicaSet.cpp
View File

@ -1,89 +0,0 @@
//
// ReplicaSet.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: ReplicaSet
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/ReplicaSet.h"
#include "Poco/MongoDB/QueryRequest.h"
#include "Poco/MongoDB/ResponseMessage.h"
namespace Poco {
namespace MongoDB {
ReplicaSet::ReplicaSet(const std::vector<Net::SocketAddress> &addresses):
_addresses(addresses)
{
}
ReplicaSet::~ReplicaSet()
{
}
Connection::Ptr ReplicaSet::findMaster()
{
Connection::Ptr master;
for (std::vector<Net::SocketAddress>::iterator it = _addresses.begin(); it != _addresses.end(); ++it)
{
master = isMaster(*it);
if (!master.isNull())
{
break;
}
}
return master;
}
Connection::Ptr ReplicaSet::isMaster(const Net::SocketAddress& address)
{
Connection::Ptr conn = new Connection();
try
{
conn->connect(address);
QueryRequest request("admin.$cmd");
request.setNumberToReturn(1);
request.selector().add("isMaster", 1);
ResponseMessage response;
conn->sendRequest(request, response);
if (response.documents().size() > 0)
{
Document::Ptr doc = response.documents()[0];
if (doc->get<bool>("ismaster"))
{
return conn;
}
else if (doc->exists("primary"))
{
return isMaster(Net::SocketAddress(doc->get<std::string>("primary")));
}
}
}
catch (...)
{
conn = 0;
}
return 0;
}
} } // namespace Poco::MongoDB

51
base/poco/MongoDB/src/RequestMessage.cpp
View File

@ -1,51 +0,0 @@
//
// RequestMessage.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: RequestMessage
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/RequestMessage.h"
#include "Poco/Net/SocketStream.h"
#include "Poco/StreamCopier.h"
namespace Poco {
namespace MongoDB {
RequestMessage::RequestMessage(MessageHeader::OpCode opcode):
Message(opcode)
{
}
RequestMessage::~RequestMessage()
{
}
void RequestMessage::send(std::ostream& ostr)
{
std::stringstream ss;
BinaryWriter requestWriter(ss, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
buildRequest(requestWriter);
requestWriter.flush();
messageLength(static_cast<Poco::Int32>(ss.tellp()));
BinaryWriter socketWriter(ostr, BinaryWriter::LITTLE_ENDIAN_BYTE_ORDER);
_header.write(socketWriter);
StreamCopier::copyStream(ss, ostr);
ostr.flush();
}
} } // namespace Poco::MongoDB

80
base/poco/MongoDB/src/ResponseMessage.cpp
View File

@ -1,80 +0,0 @@
//
// ResponseMessage.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: ResponseMessage
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/ResponseMessage.h"
#include "Poco/Net/SocketStream.h"
namespace Poco {
namespace MongoDB {
ResponseMessage::ResponseMessage():
Message(MessageHeader::OP_REPLY),
_responseFlags(0),
_cursorID(0),
_startingFrom(0),
_numberReturned(0)
{
}
ResponseMessage::ResponseMessage(const Int64& cursorID):
Message(MessageHeader::OP_REPLY),
_responseFlags(0),
_cursorID(cursorID),
_startingFrom(0),
_numberReturned(0)
{
}
ResponseMessage::~ResponseMessage()
{
}
void ResponseMessage::clear()
{
_responseFlags = 0;
_startingFrom = 0;
_cursorID = 0;
_numberReturned = 0;
_documents.clear();
}
void ResponseMessage::read(std::istream& istr)
{
clear();
BinaryReader reader(istr, BinaryReader::LITTLE_ENDIAN_BYTE_ORDER);
_header.read(reader);
reader >> _responseFlags;
reader >> _cursorID;
reader >> _startingFrom;
reader >> _numberReturned;
for (int i = 0; i < _numberReturned; ++i)
{
Document::Ptr doc = new Document();
doc->read(reader);
_documents.push_back(doc);
}
}
} } // namespace Poco::MongoDB

47
base/poco/MongoDB/src/UpdateRequest.cpp
View File

@ -1,47 +0,0 @@
//
// UpdateRequest.cpp
//
// Library: MongoDB
// Package: MongoDB
// Module: UpdateRequest
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MongoDB/UpdateRequest.h"
namespace Poco {
namespace MongoDB {
UpdateRequest::UpdateRequest(const std::string& collectionName, UpdateRequest::Flags flags):
RequestMessage(MessageHeader::OP_UPDATE),
_flags(flags),
_fullCollectionName(collectionName),
_selector(),
_update()
{
}
UpdateRequest::~UpdateRequest()
{
}
void UpdateRequest::buildRequest(BinaryWriter& writer)
{
writer << 0; // 0 - reserved for future use
BSONWriter(writer).writeCString(_fullCollectionName);
writer << _flags;
_selector.write(writer);
_update.write(writer);
}
} } // namespace Poco::MongoDB

5
docs/en/engines/table-engines/integrations/mongodb.md
View File

@ -11,11 +11,6 @@ MongoDB engine is read-only table engine which allows to read data from remote [
Only MongoDB v3.6+ servers are supported.
[Seed list(`mongodb+srv`)](https://www.mongodb.com/docs/manual/reference/glossary/#std-term-seed-list) is not yet supported.
:::note
If you're facing troubles, please report the issue, and try to use [the legacy implementation](../../../operations/server-configuration-parameters/settings.md#use_legacy_mongodb_integration).
Keep in mind that it is deprecated, and will be removed in next releases.
:::
## Creating a Table {#creating-a-table}
``` sql

8
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -3277,11 +3277,3 @@ Type: UInt64
Default value: 100
Zero means unlimited
## use_legacy_mongodb_integration
Use the legacy MongoDB integration implementation. Deprecated.
Type: Bool
Default value: `true`.

4
programs/format/Format.cpp
View File

@ -188,9 +188,9 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
registerInterpreters();
registerFunctions();
registerAggregateFunctions();
registerTableFunctions(false);
registerTableFunctions();
registerDatabases();
registerStorages(false);
registerStorages();
registerFormats();
std::unordered_set<std::string> additional_names;

7
programs/local/LocalServer.cpp
View File

@ -110,7 +110,6 @@ namespace ServerSetting
extern const ServerSettingsString uncompressed_cache_policy;
extern const ServerSettingsUInt64 uncompressed_cache_size;
extern const ServerSettingsDouble uncompressed_cache_size_ratio;
extern const ServerSettingsBool use_legacy_mongodb_integration;
}
namespace ErrorCodes
@ -549,10 +548,10 @@ try
/// Don't initialize DateLUT
registerFunctions();
registerAggregateFunctions();
registerTableFunctions(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerTableFunctions();
registerDatabases();
registerStorages(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerDictionaries(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerStorages();
registerDictionaries();
registerDisks(/* global_skip_access_check= */ true);
registerFormats();

7
programs/server/Server.cpp
View File

@ -279,7 +279,6 @@ namespace ServerSetting
extern const ServerSettingsString uncompressed_cache_policy;
extern const ServerSettingsUInt64 uncompressed_cache_size;
extern const ServerSettingsDouble uncompressed_cache_size_ratio;
extern const ServerSettingsBool use_legacy_mongodb_integration;
}
}
@ -912,10 +911,10 @@ try
registerInterpreters();
registerFunctions();
registerAggregateFunctions();
registerTableFunctions(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerTableFunctions();
registerDatabases();
registerStorages(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerDictionaries(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerStorages();
registerDictionaries();
registerDisks(/* global_skip_access_check= */ false);
registerFormats();
registerRemoteFileMetadatas();

4
src/CMakeLists.txt
View File

@ -433,10 +433,6 @@ dbms_target_link_libraries (
Poco::Redis
)
if (USE_MONGODB)
dbms_target_link_libraries (PUBLIC Poco::MongoDB)
endif()
if (TARGET ch_contrib::mongocxx)
dbms_target_link_libraries(
PUBLIC

1
src/Core/ServerSettings.cpp
View File

@ -194,7 +194,6 @@ namespace DB
DECLARE(UInt64, parts_kill_delay_period_random_add, 10, "Add uniformly distributed value from 0 to x seconds to kill_delay_period to avoid thundering herd effect and subsequent DoS of ZooKeeper in case of very large number of tables. Only available in ClickHouse Cloud", 0) \
DECLARE(UInt64, parts_killer_pool_size, 128, "Threads for cleanup of shared merge tree outdated threads. Only available in ClickHouse Cloud", 0) \
DECLARE(UInt64, keeper_multiread_batch_size, 10'000, "Maximum size of batch for MultiRead request to [Zoo]Keeper that support batching. If set to 0, batching is disabled. Available only in ClickHouse Cloud.", 0) \
DECLARE(Bool, use_legacy_mongodb_integration, true, "Use the legacy MongoDB integration implementation. Note: it's highly recommended to set this option to false, since legacy implementation will be removed in the future. Please submit any issues you encounter with the new implementation.", 0) \
\
DECLARE(UInt64, prefetch_threadpool_pool_size, 100, "Size of background pool for prefetches for remote object storages", 0) \
DECLARE(UInt64, prefetch_threadpool_queue_size, 1000000, "Number of tasks which is possible to push into prefetches pool", 0) \

2
src/Core/Settings.cpp
View File

@ -5715,7 +5715,7 @@ Enable `IF NOT EXISTS` for `CREATE` statement by default. If either this setting
If enabled, only allow identifiers containing alphanumeric characters and underscores.
)", 0) \
DECLARE(Bool, mongodb_throw_on_unsupported_query, true, R"(
If enabled, MongoDB tables will return an error when a MongoDB query cannot be built. Otherwise, ClickHouse reads the full table and processes it locally. This option does not apply to the legacy implementation or when 'allow_experimental_analyzer=0'.
If enabled, MongoDB tables will return an error when a MongoDB query cannot be built. Otherwise, ClickHouse reads the full table and processes it locally. This option is not applied when 'allow_experimental_analyzer=0'.
)", 0) \
DECLARE(Bool, implicit_select, false, R"(
Allow writing simple SELECT queries without the leading SELECT keyword, which makes it simple for calculator-style usage, e.g. `1 + 2` becomes a valid query.

4
src/Dictionaries/CMakeLists.txt
View File

@ -40,10 +40,6 @@ target_link_libraries(clickhouse_dictionaries
Poco::Redis
)
if (USE_MONGODB)
target_link_libraries(clickhouse_dictionaries PRIVATE Poco::MongoDB)
endif()
target_link_libraries(clickhouse_dictionaries PUBLIC ch_contrib::abseil_swiss_tables)
if (TARGET ch_contrib::cassandra)

305
src/Dictionaries/MongoDBPocoLegacyDictionarySource.cpp
View File

@ -1,305 +0,0 @@
#include "config.h"
#include "DictionarySourceFactory.h"
#if USE_MONGODB
#include <Common/RemoteHostFilter.h>
#include "MongoDBPocoLegacyDictionarySource.h"
#include "DictionaryStructure.h"
#include "registerDictionaries.h"
#include <Storages/StorageMongoDBPocoLegacySocketFactory.h>
#include <Storages/NamedCollectionsHelpers.h>
#endif
namespace DB
{
namespace ErrorCodes
{
#if USE_MONGODB
extern const int NOT_IMPLEMENTED;
extern const int UNSUPPORTED_METHOD;
extern const int MONGODB_CANNOT_AUTHENTICATE;
#else
extern const int SUPPORT_IS_DISABLED;
#endif
}
void registerDictionarySourceMongoDBPocoLegacy(DictionarySourceFactory & factory)
{
#if USE_MONGODB
auto create_mongo_db_dictionary = [](
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & root_config_prefix,
Block & sample_block,
ContextPtr context,
const std::string & /* default_database */,
bool created_from_ddl)
{
const auto config_prefix = root_config_prefix + ".mongodb";
auto named_collection = created_from_ddl ? tryGetNamedCollectionWithOverrides(config, config_prefix, context) : nullptr;
String host, username, password, database, method, options, collection;
UInt16 port;
if (named_collection)
{
validateNamedCollection(
*named_collection,
/* required_keys */{"collection"},
/* optional_keys */ValidateKeysMultiset<ExternalDatabaseEqualKeysSet>{
"host", "port", "user", "password", "db", "database", "uri", "name", "method", "options"});
host = named_collection->getOrDefault<String>("host", "");
port = static_cast<UInt16>(named_collection->getOrDefault<UInt64>("port", 0));
username = named_collection->getOrDefault<String>("user", "");
password = named_collection->getOrDefault<String>("password", "");
database = named_collection->getAnyOrDefault<String>({"db", "database"}, "");
method = named_collection->getOrDefault<String>("method", "");
collection = named_collection->getOrDefault<String>("collection", "");
options = named_collection->getOrDefault<String>("options", "");
}
else
{
host = config.getString(config_prefix + ".host", "");
port = config.getUInt(config_prefix + ".port", 0);
username = config.getString(config_prefix + ".user", "");
password = config.getString(config_prefix + ".password", "");
database = config.getString(config_prefix + ".db", "");
method = config.getString(config_prefix + ".method", "");
collection = config.getString(config_prefix + ".collection");
options = config.getString(config_prefix + ".options", "");
}
if (created_from_ddl)
context->getRemoteHostFilter().checkHostAndPort(host, toString(port));
return std::make_unique<MongoDBPocoLegacyDictionarySource>(dict_struct,
config.getString(config_prefix + ".uri", ""),
host,
port,
username,
password,
method,
database,
collection,
options,
sample_block);
};
#else
auto create_mongo_db_dictionary = [](
const DictionaryStructure & /* dict_struct */,
const Poco::Util::AbstractConfiguration & /* config */,
const std::string & /* root_config_prefix */,
Block & /* sample_block */,
ContextPtr /* context */,
const std::string & /* default_database */,
bool /* created_from_ddl */) -> DictionarySourcePtr
{
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
"Dictionary source of type `mongodb` is disabled because ClickHouse was built without mongodb support.");
};
#endif
factory.registerSource("mongodb", create_mongo_db_dictionary);
}
}
#if USE_MONGODB
#include <Common/logger_useful.h>
#include <Poco/MongoDB/Array.h>
#include <Poco/MongoDB/Connection.h>
#include <Poco/MongoDB/Cursor.h>
#include <Poco/MongoDB/Database.h>
#include <Poco/MongoDB/ObjectId.h>
#include <Poco/URI.h>
#include <Poco/Util/AbstractConfiguration.h>
// only after poco
// naming conflict:
// Poco/MongoDB/BSONWriter.h:54: void writeCString(const std::string & value);
// src/IO/WriteHelpers.h:146 #define writeCString(s, buf)
#include <IO/WriteHelpers.h>
namespace DB
{
static const UInt64 max_block_size = 8192;
MongoDBPocoLegacyDictionarySource::MongoDBPocoLegacyDictionarySource(
const DictionaryStructure & dict_struct_,
const std::string & uri_,
const std::string & host_,
UInt16 port_,
const std::string & user_,
const std::string & password_,
const std::string & method_,
const std::string & db_,
const std::string & collection_,
const std::string & options_,
const Block & sample_block_)
: dict_struct{dict_struct_}
, uri{uri_}
, host{host_}
, port{port_}
, user{user_}
, password{password_}
, method{method_}
, db{db_}
, collection{collection_}
, options(options_)
, sample_block{sample_block_}
, connection{std::make_shared<Poco::MongoDB::Connection>()}
{
StorageMongoDBPocoLegacySocketFactory socket_factory;
if (!uri.empty())
{
// Connect with URI.
connection->connect(uri, socket_factory);
Poco::URI poco_uri(connection->uri());
// Parse database from URI. This is required for correctness -- the
// cursor is created using database name and collection name, so we have
// to specify them properly.
db = poco_uri.getPath();
// getPath() may return a leading slash, remove it.
if (!db.empty() && db[0] == '/')
{
db.erase(0, 1);
}
// Parse some other parts from URI, for logging and display purposes.
host = poco_uri.getHost();
port = poco_uri.getPort();
user = poco_uri.getUserInfo();
if (size_t separator = user.find(':'); separator != std::string::npos)
{
user.resize(separator);
}
}
else
{
// Connect with host/port/user/etc through constructing the uri
std::string uri_constructed("mongodb://" + host + ":" + std::to_string(port) + "/" + db + (options.empty() ? "" : "?" + options));
connection->connect(uri_constructed, socket_factory);
if (!user.empty())
{
Poco::MongoDB::Database poco_db(db);
if (!poco_db.authenticate(*connection, user, password, method.empty() ? Poco::MongoDB::Database::AUTH_SCRAM_SHA1 : method))
throw Exception(ErrorCodes::MONGODB_CANNOT_AUTHENTICATE, "Cannot authenticate in MongoDB, incorrect user or password");
}
}
}
MongoDBPocoLegacyDictionarySource::MongoDBPocoLegacyDictionarySource(const MongoDBPocoLegacyDictionarySource & other)
: MongoDBPocoLegacyDictionarySource{
other.dict_struct, other.uri, other.host, other.port, other.user, other.password, other.method, other.db,
other.collection, other.options, other.sample_block
}
{
}
MongoDBPocoLegacyDictionarySource::~MongoDBPocoLegacyDictionarySource() = default;
QueryPipeline MongoDBPocoLegacyDictionarySource::loadAll()
{
return QueryPipeline(std::make_shared<MongoDBPocoLegacySource>(connection, db, collection, Poco::MongoDB::Document{}, sample_block, max_block_size));
}
QueryPipeline MongoDBPocoLegacyDictionarySource::loadIds(const std::vector<UInt64> & ids)
{
if (!dict_struct.id)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "'id' is required for selective loading");
Poco::MongoDB::Document query;
/** NOTE: While building array, Poco::MongoDB requires passing of different unused element names, along with values.
* In general, Poco::MongoDB is quite inefficient and bulky.
*/
Poco::MongoDB::Array::Ptr ids_array(new Poco::MongoDB::Array);
for (const UInt64 id : ids)
ids_array->add(DB::toString(id), static_cast<Int32>(id));
query.addNewDocument(dict_struct.id->name).add("$in", ids_array);
return QueryPipeline(std::make_shared<MongoDBPocoLegacySource>(connection, db, collection, query, sample_block, max_block_size));
}
QueryPipeline MongoDBPocoLegacyDictionarySource::loadKeys(const Columns & key_columns, const std::vector<size_t> & requested_rows)
{
if (!dict_struct.key)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "'key' is required for selective loading");
Poco::MongoDB::Document query;
Poco::MongoDB::Array::Ptr keys_array(new Poco::MongoDB::Array);
for (const auto row_idx : requested_rows)
{
auto & key = keys_array->addNewDocument(DB::toString(row_idx));
const auto & key_attributes = *dict_struct.key;
for (size_t attribute_index = 0; attribute_index < key_attributes.size(); ++attribute_index)
{
const auto & key_attribute = key_attributes[attribute_index];
switch (key_attribute.underlying_type)
{
case AttributeUnderlyingType::UInt8:
case AttributeUnderlyingType::UInt16:
case AttributeUnderlyingType::UInt32:
case AttributeUnderlyingType::UInt64:
case AttributeUnderlyingType::Int8:
case AttributeUnderlyingType::Int16:
case AttributeUnderlyingType::Int32:
case AttributeUnderlyingType::Int64:
{
key.add(key_attribute.name, static_cast<Int32>(key_columns[attribute_index]->get64(row_idx)));
break;
}
case AttributeUnderlyingType::Float32:
case AttributeUnderlyingType::Float64:
{
key.add(key_attribute.name, key_columns[attribute_index]->getFloat64(row_idx));
break;
}
case AttributeUnderlyingType::String:
{
String loaded_str((*key_columns[attribute_index])[row_idx].safeGet<String>());
/// Convert string to ObjectID
if (key_attribute.is_object_id)
{
Poco::MongoDB::ObjectId::Ptr loaded_id(new Poco::MongoDB::ObjectId(loaded_str));
key.add(key_attribute.name, loaded_id);
}
else
{
key.add(key_attribute.name, loaded_str);
}
break;
}
default:
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported dictionary attribute type for MongoDB dictionary source");
}
}
}
/// If more than one key we should use $or
query.add("$or", keys_array);
return QueryPipeline(std::make_shared<MongoDBPocoLegacySource>(connection, db, collection, query, sample_block, max_block_size));
}
std::string MongoDBPocoLegacyDictionarySource::toString() const
{
return fmt::format("MongoDB: {}.{},{}{}:{}", db, collection, (user.empty() ? " " : " " + user + '@'), host, port);
}
}
#endif

93
src/Dictionaries/MongoDBPocoLegacyDictionarySource.h
View File

@ -1,93 +0,0 @@
#pragma once
#include "config.h"
#if USE_MONGODB
#include <Processors/Sources/MongoDBPocoLegacySource.h>
#include <Core/Block.h>
#include "DictionaryStructure.h"
#include "IDictionarySource.h"
namespace Poco
{
namespace Util
{
class AbstractConfiguration;
}
namespace MongoDB
{
class Connection;
}
}
namespace DB
{
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
/// Allows loading dictionaries from a MongoDB collection. Deprecated, will be removed soon.
class MongoDBPocoLegacyDictionarySource final : public IDictionarySource
{
public:
MongoDBPocoLegacyDictionarySource(
const DictionaryStructure & dict_struct_,
const std::string & uri_,
const std::string & host_,
UInt16 port_,
const std::string & user_,
const std::string & password_,
const std::string & method_,
const std::string & db_,
const std::string & collection_,
const std::string & options,
const Block & sample_block_);
MongoDBPocoLegacyDictionarySource(const MongoDBPocoLegacyDictionarySource & other);
~MongoDBPocoLegacyDictionarySource() override;
QueryPipeline loadAll() override;
QueryPipeline loadUpdatedAll() override
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method loadUpdatedAll is unsupported for MongoDBDictionarySource");
}
bool supportsSelectiveLoad() const override { return true; }
QueryPipeline loadIds(const std::vector<UInt64> & ids) override;
QueryPipeline loadKeys(const Columns & key_columns, const std::vector<size_t> & requested_rows) override;
/// @todo: for MongoDB, modification date can somehow be determined from the `_id` object field
bool isModified() const override { return true; }
///Not yet supported
bool hasUpdateField() const override { return false; }
DictionarySourcePtr clone() const override { return std::make_shared<MongoDBPocoLegacyDictionarySource>(*this); }
std::string toString() const override;
private:
const DictionaryStructure dict_struct;
const std::string uri;
std::string host;
UInt16 port;
std::string user;
const std::string password;
const std::string method;
std::string db;
const std::string collection;
const std::string options;
Block sample_block;
std::shared_ptr<Poco::MongoDB::Connection> connection;
};
}
#endif

7
src/Dictionaries/registerDictionaries.cpp
View File

@ -36,7 +36,7 @@ void registerDictionaryPolygon(DictionaryFactory & factory);
void registerDictionaryDirect(DictionaryFactory & factory);
void registerDictionaries(bool use_legacy_mongodb_integration)
void registerDictionaries()
{
{
auto & source_factory = DictionarySourceFactory::instance();
@ -45,10 +45,7 @@ void registerDictionaries(bool use_legacy_mongodb_integration)
registerDictionarySourceMysql(source_factory);
registerDictionarySourceClickHouse(source_factory);
if (use_legacy_mongodb_integration)
registerDictionarySourceMongoDBPocoLegacy(source_factory);
else
registerDictionarySourceMongoDB(source_factory);
registerDictionarySourceMongoDB(source_factory);
registerDictionarySourceRedis(source_factory);
registerDictionarySourceCassandra(source_factory);

2
src/Dictionaries/registerDictionaries.h
View File

@ -2,5 +2,5 @@
namespace DB
{
void registerDictionaries(bool use_legacy_mongodb_integration);
void registerDictionaries();
}

578
src/Processors/Sources/MongoDBPocoLegacySource.cpp
View File

@ -1,578 +0,0 @@
#include "config.h"
#if USE_MONGODB
#include "MongoDBPocoLegacySource.h"
#include <string>
#include <vector>
#include <Poco/MongoDB/Array.h>
#include <Poco/MongoDB/Binary.h>
#include <Poco/MongoDB/Database.h>
#include <Poco/MongoDB/Connection.h>
#include <Poco/MongoDB/Cursor.h>
#include <Poco/MongoDB/OpMsgCursor.h>
#include <Poco/MongoDB/ObjectId.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnsNumber.h>
#include <IO/ReadHelpers.h>
#include <Common/assert_cast.h>
#include <Common/quoteString.h>
#include "base/types.h"
#include <base/range.h>
#include <Poco/URI.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeNullable.h>
// only after poco
// naming conflict:
// Poco/MongoDB/BSONWriter.h:54: void writeCString(const std::string & value);
// src/IO/WriteHelpers.h:146 #define writeCString(s, buf)
#include <IO/WriteHelpers.h>
namespace DB
{
namespace ErrorCodes
{
extern const int TYPE_MISMATCH;
extern const int UNKNOWN_TYPE;
extern const int MONGODB_ERROR;
extern const int BAD_ARGUMENTS;
}
namespace
{
using ValueType = ExternalResultDescription::ValueType;
using ObjectId = Poco::MongoDB::ObjectId;
using MongoArray = Poco::MongoDB::Array;
using MongoUUID = Poco::MongoDB::Binary::Ptr;
UUID parsePocoUUID(const Poco::UUID & src)
{
UUID uuid;
std::array<Poco::UInt8, 6> src_node = src.getNode();
UInt64 node = 0;
node |= UInt64(src_node[0]) << 40;
node |= UInt64(src_node[1]) << 32;
node |= UInt64(src_node[2]) << 24;
node |= UInt64(src_node[3]) << 16;
node |= UInt64(src_node[4]) << 8;
node |= src_node[5];
UUIDHelpers::getHighBytes(uuid) = UInt64(src.getTimeLow()) << 32 | UInt32(src.getTimeMid() << 16 | src.getTimeHiAndVersion());
UUIDHelpers::getLowBytes(uuid) = UInt64(src.getClockSeq()) << 48 | node;
return uuid;
}
template <typename T>
Field getNumber(const Poco::MongoDB::Element & value, const std::string & name)
{
switch (value.type())
{
case Poco::MongoDB::ElementTraits<Int32>::TypeId:
return static_cast<T>(static_cast<const Poco::MongoDB::ConcreteElement<Int32> &>(value).value());
case Poco::MongoDB::ElementTraits<Poco::Int64>::TypeId:
return static_cast<T>(static_cast<const Poco::MongoDB::ConcreteElement<Poco::Int64> &>(value).value());
case Poco::MongoDB::ElementTraits<Float64>::TypeId:
return static_cast<T>(static_cast<const Poco::MongoDB::ConcreteElement<Float64> &>(value).value());
case Poco::MongoDB::ElementTraits<bool>::TypeId:
return static_cast<T>(static_cast<const Poco::MongoDB::ConcreteElement<bool> &>(value).value());
case Poco::MongoDB::ElementTraits<Poco::MongoDB::NullValue>::TypeId:
return Field();
case Poco::MongoDB::ElementTraits<String>::TypeId:
return parse<T>(static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value());
default:
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected a number, got type id = {} for column {}",
toString(value.type()), name);
}
}
void prepareMongoDBArrayInfo(
std::unordered_map<size_t, MongoDBPocoLegacyArrayInfo> & array_info, size_t column_idx, const DataTypePtr data_type)
{
const auto * array_type = assert_cast<const DataTypeArray *>(data_type.get());
auto nested = array_type->getNestedType();
size_t count_dimensions = 1;
while (isArray(nested))
{
++count_dimensions;
nested = assert_cast<const DataTypeArray *>(nested.get())->getNestedType();
}
Field default_value = nested->getDefault();
if (nested->isNullable())
nested = assert_cast<const DataTypeNullable *>(nested.get())->getNestedType();
WhichDataType which(nested);
std::function<Field(const Poco::MongoDB::Element & value, const std::string & name)> parser;
if (which.isUInt8())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<UInt8>(value, name); };
else if (which.isUInt16())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<UInt16>(value, name); };
else if (which.isUInt32())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<UInt32>(value, name); };
else if (which.isUInt64())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<UInt64>(value, name); };
else if (which.isInt8())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<Int8>(value, name); };
else if (which.isInt16())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<Int16>(value, name); };
else if (which.isInt32())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<Int32>(value, name); };
else if (which.isInt64())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<Int64>(value, name); };
else if (which.isFloat32())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<Float32>(value, name); };
else if (which.isFloat64())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber<Float64>(value, name); };
else if (which.isString() || which.isFixedString())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field
{
if (value.type() == Poco::MongoDB::ElementTraits<ObjectId::Ptr>::TypeId)
{
String string_id = value.toString();
return Field(string_id.data(), string_id.size());
}
if (value.type() == Poco::MongoDB::ElementTraits<String>::TypeId)
{
String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
return Field(string.data(), string.size());
}
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String, got type id = {} for column {}",
toString(value.type()), name);
};
else if (which.isDate())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field
{
if (value.type() != Poco::MongoDB::ElementTraits<Poco::Timestamp>::TypeId)
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected Timestamp, got type id = {} for column {}",
toString(value.type()), name);
return static_cast<UInt16>(DateLUT::instance().toDayNum(
static_cast<const Poco::MongoDB::ConcreteElement<Poco::Timestamp> &>(value).value().epochTime()));
};
else if (which.isDateTime())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field
{
if (value.type() != Poco::MongoDB::ElementTraits<Poco::Timestamp>::TypeId)
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected Timestamp, got type id = {} for column {}",
toString(value.type()), name);
return static_cast<UInt32>(static_cast<const Poco::MongoDB::ConcreteElement<Poco::Timestamp> &>(value).value().epochTime());
};
else if (which.isUUID())
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field
{
if (value.type() == Poco::MongoDB::ElementTraits<String>::TypeId)
{
String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
return parse<UUID>(string);
}
if (value.type() == Poco::MongoDB::ElementTraits<MongoUUID>::TypeId)
{
const Poco::UUID & poco_uuid = static_cast<const Poco::MongoDB::ConcreteElement<MongoUUID> &>(value).value()->uuid();
return parsePocoUUID(poco_uuid);
}
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String/UUID, got type id = {} for column {}",
toString(value.type()), name);
};
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Type conversion to {} is not supported", nested->getName());
array_info[column_idx] = {count_dimensions, default_value, parser};
}
template <typename T>
void insertNumber(IColumn & column, const Poco::MongoDB::Element & value, const std::string & name)
{
switch (value.type())
{
case Poco::MongoDB::ElementTraits<Int32>::TypeId:
assert_cast<ColumnVector<T> &>(column).getData().push_back(
static_cast<const Poco::MongoDB::ConcreteElement<Int32> &>(value).value());
break;
case Poco::MongoDB::ElementTraits<Poco::Int64>::TypeId:
assert_cast<ColumnVector<T> &>(column).getData().push_back(
static_cast<T>(static_cast<const Poco::MongoDB::ConcreteElement<Poco::Int64> &>(value).value()));
break;
case Poco::MongoDB::ElementTraits<Float64>::TypeId:
assert_cast<ColumnVector<T> &>(column).getData().push_back(static_cast<T>(
static_cast<const Poco::MongoDB::ConcreteElement<Float64> &>(value).value()));
break;
case Poco::MongoDB::ElementTraits<bool>::TypeId:
assert_cast<ColumnVector<T> &>(column).getData().push_back(
static_cast<const Poco::MongoDB::ConcreteElement<bool> &>(value).value());
break;
case Poco::MongoDB::ElementTraits<Poco::MongoDB::NullValue>::TypeId:
assert_cast<ColumnVector<T> &>(column).getData().emplace_back();
break;
case Poco::MongoDB::ElementTraits<String>::TypeId:
assert_cast<ColumnVector<T> &>(column).getData().push_back(
parse<T>(static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value()));
break;
default:
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected a number, got type id = {} for column {}",
toString(value.type()), name);
}
}
void insertValue(
IColumn & column,
const ValueType type,
const Poco::MongoDB::Element & value,
const std::string & name,
std::unordered_map<size_t, MongoDBPocoLegacyArrayInfo> & array_info,
size_t idx)
{
switch (type)
{
case ValueType::vtUInt8:
insertNumber<UInt8>(column, value, name);
break;
case ValueType::vtUInt16:
insertNumber<UInt16>(column, value, name);
break;
case ValueType::vtUInt32:
insertNumber<UInt32>(column, value, name);
break;
case ValueType::vtUInt64:
insertNumber<UInt64>(column, value, name);
break;
case ValueType::vtInt8:
insertNumber<Int8>(column, value, name);
break;
case ValueType::vtInt16:
insertNumber<Int16>(column, value, name);
break;
case ValueType::vtInt32:
insertNumber<Int32>(column, value, name);
break;
case ValueType::vtInt64:
insertNumber<Int64>(column, value, name);
break;
case ValueType::vtFloat32:
insertNumber<Float32>(column, value, name);
break;
case ValueType::vtFloat64:
insertNumber<Float64>(column, value, name);
break;
case ValueType::vtEnum8:
case ValueType::vtEnum16:
case ValueType::vtString:
{
if (value.type() == Poco::MongoDB::ElementTraits<ObjectId::Ptr>::TypeId)
{
std::string string_id = value.toString();
assert_cast<ColumnString &>(column).insertData(string_id.data(), string_id.size());
break;
}
if (value.type() == Poco::MongoDB::ElementTraits<String>::TypeId)
{
String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
assert_cast<ColumnString &>(column).insertData(string.data(), string.size());
break;
}
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String, got type id = {} for column {}",
toString(value.type()), name);
}
case ValueType::vtDate:
{
if (value.type() != Poco::MongoDB::ElementTraits<Poco::Timestamp>::TypeId)
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected Timestamp, got type id = {} for column {}",
toString(value.type()), name);
assert_cast<ColumnUInt16 &>(column).getData().push_back(static_cast<UInt16>(DateLUT::instance().toDayNum(
static_cast<const Poco::MongoDB::ConcreteElement<Poco::Timestamp> &>(value).value().epochTime())));
break;
}
case ValueType::vtDateTime:
{
if (value.type() != Poco::MongoDB::ElementTraits<Poco::Timestamp>::TypeId)
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected Timestamp, got type id = {} for column {}",
toString(value.type()), name);
assert_cast<ColumnUInt32 &>(column).getData().push_back(
static_cast<UInt32>(static_cast<const Poco::MongoDB::ConcreteElement<Poco::Timestamp> &>(value).value().epochTime()));
break;
}
case ValueType::vtUUID:
{
if (value.type() == Poco::MongoDB::ElementTraits<String>::TypeId)
{
String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
assert_cast<ColumnUUID &>(column).getData().push_back(parse<UUID>(string));
}
else if (value.type() == Poco::MongoDB::ElementTraits<MongoUUID>::TypeId)
{
const Poco::UUID & poco_uuid = static_cast<const Poco::MongoDB::ConcreteElement<MongoUUID> &>(value).value()->uuid();
UUID uuid = parsePocoUUID(poco_uuid);
assert_cast<ColumnUUID &>(column).getData().push_back(uuid);
}
else
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String/UUID, got type id = {} for column {}",
toString(value.type()), name);
break;
}
case ValueType::vtArray:
{
if (value.type() != Poco::MongoDB::ElementTraits<MongoArray::Ptr>::TypeId)
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected Array, got type id = {} for column {}",
toString(value.type()), name);
size_t expected_dimensions = array_info[idx].num_dimensions;
const auto parse_value = array_info[idx].parser;
std::vector<Row> dimensions(expected_dimensions + 1);
auto array = static_cast<const Poco::MongoDB::ConcreteElement<MongoArray::Ptr> &>(value).value();
std::vector<std::pair<const Poco::MongoDB::Element *, size_t>> arrays;
arrays.emplace_back(&value, 0);
while (!arrays.empty())
{
size_t dimension_idx = arrays.size() - 1;
if (dimension_idx + 1 > expected_dimensions)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Got more dimensions than expected");
auto [parent_ptr, child_idx] = arrays.back();
auto parent = static_cast<const Poco::MongoDB::ConcreteElement<MongoArray::Ptr> &>(*parent_ptr).value();
if (child_idx >= parent->size())
{
arrays.pop_back();
if (dimension_idx == 0)
break;
dimensions[dimension_idx].emplace_back(Array(dimensions[dimension_idx + 1].begin(), dimensions[dimension_idx + 1].end()));
dimensions[dimension_idx + 1].clear();
continue;
}
Poco::MongoDB::Element::Ptr child = parent->get(static_cast<int>(child_idx));
arrays.back().second += 1;
if (child->type() == Poco::MongoDB::ElementTraits<MongoArray::Ptr>::TypeId)
{
arrays.emplace_back(child.get(), 0);
}
else if (child->type() == Poco::MongoDB::ElementTraits<Poco::MongoDB::NullValue>::TypeId)
{
if (dimension_idx + 1 == expected_dimensions)
dimensions[dimension_idx + 1].emplace_back(array_info[idx].default_value);
else
dimensions[dimension_idx + 1].emplace_back(Array());
}
else if (dimension_idx + 1 == expected_dimensions)
{
dimensions[dimension_idx + 1].emplace_back(parse_value(*child, name));
}
else
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Got less dimensions than expected. ({} instead of {})", dimension_idx + 1, expected_dimensions);
}
}
assert_cast<ColumnArray &>(column).insert(Array(dimensions[1].begin(), dimensions[1].end()));
break;
}
default:
throw Exception(ErrorCodes::UNKNOWN_TYPE, "Value of unsupported type: {}", column.getName());
}
}
void insertDefaultValue(IColumn & column, const IColumn & sample_column) { column.insertFrom(sample_column, 0); }
}
bool isMongoDBWireProtocolOld(Poco::MongoDB::Connection & connection_, const std::string & database_name_)
{
Poco::MongoDB::Database db(database_name_);
Poco::MongoDB::Document::Ptr doc = db.queryServerHello(connection_, false);
if (doc->exists("maxWireVersion"))
{
auto wire_version = doc->getInteger("maxWireVersion");
return wire_version < Poco::MongoDB::Database::WireVersion::VER_36;
}
doc = db.queryServerHello(connection_, true);
if (doc->exists("maxWireVersion"))
{
auto wire_version = doc->getInteger("maxWireVersion");
return wire_version < Poco::MongoDB::Database::WireVersion::VER_36;
}
return true;
}
MongoDBPocoLegacyCursor::MongoDBPocoLegacyCursor(
const std::string & database,
const std::string & collection,
const Block & sample_block_to_select,
const Poco::MongoDB::Document & query,
Poco::MongoDB::Connection & connection)
: is_wire_protocol_old(isMongoDBWireProtocolOld(connection, database))
{
Poco::MongoDB::Document projection;
/// Looks like selecting _id column is implicit by default.
if (!sample_block_to_select.has("_id"))
projection.add("_id", 0);
for (const auto & column : sample_block_to_select)
projection.add(column.name, 1);
if (is_wire_protocol_old)
{
old_cursor = std::make_unique<Poco::MongoDB::Cursor>(database, collection);
old_cursor->query().selector() = query;
old_cursor->query().returnFieldSelector() = projection;
}
else
{
new_cursor = std::make_unique<Poco::MongoDB::OpMsgCursor>(database, collection);
new_cursor->query().setCommandName(Poco::MongoDB::OpMsgMessage::CMD_FIND);
new_cursor->query().body().addNewDocument("filter") = query;
new_cursor->query().body().addNewDocument("projection") = projection;
}
}
Poco::MongoDB::Document::Vector MongoDBPocoLegacyCursor::nextDocuments(Poco::MongoDB::Connection & connection)
{
if (is_wire_protocol_old)
{
auto response = old_cursor->next(connection);
cursor_id = response.cursorID();
return std::move(response.documents());
}
auto response = new_cursor->next(connection);
cursor_id = new_cursor->cursorID();
return std::move(response.documents());
}
Int64 MongoDBPocoLegacyCursor::cursorID() const
{
return cursor_id;
}
MongoDBPocoLegacySource::MongoDBPocoLegacySource(
std::shared_ptr<Poco::MongoDB::Connection> & connection_,
const String & database_name_,
const String & collection_name_,
const Poco::MongoDB::Document & query_,
const Block & sample_block,
UInt64 max_block_size_)
: ISource(sample_block.cloneEmpty())
, connection(connection_)
, cursor(database_name_, collection_name_, sample_block, query_, *connection_)
, max_block_size{max_block_size_}
{
description.init(sample_block);
for (const auto idx : collections::range(0, description.sample_block.columns()))
if (description.types[idx].first == ExternalResultDescription::ValueType::vtArray)
prepareMongoDBArrayInfo(array_info, idx, description.sample_block.getByPosition(idx).type);
}
MongoDBPocoLegacySource::~MongoDBPocoLegacySource() = default;
Chunk MongoDBPocoLegacySource::generate()
{
if (all_read)
return {};
MutableColumns columns(description.sample_block.columns());
const size_t size = columns.size();
for (const auto i : collections::range(0, size))
columns[i] = description.sample_block.getByPosition(i).column->cloneEmpty();
size_t num_rows = 0;
while (num_rows < max_block_size)
{
auto documents = cursor.nextDocuments(*connection);
for (auto & document : documents)
{
if (document->exists("ok") && document->exists("$err")
&& document->exists("code") && document->getInteger("ok") == 0)
{
auto code = document->getInteger("code");
const Poco::MongoDB::Element::Ptr value = document->get("$err");
auto message = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(*value).value();
throw Exception(ErrorCodes::MONGODB_ERROR, "Got error from MongoDB: {}, code: {}", message, code);
}
++num_rows;
for (const auto idx : collections::range(0, size))
{
const auto & name = description.sample_block.getByPosition(idx).name;
bool exists_in_current_document = document->exists(name);
if (!exists_in_current_document)
{
insertDefaultValue(*columns[idx], *description.sample_block.getByPosition(idx).column);
continue;
}
const Poco::MongoDB::Element::Ptr value = document->get(name);
if (value.isNull() || value->type() == Poco::MongoDB::ElementTraits<Poco::MongoDB::NullValue>::TypeId)
{
insertDefaultValue(*columns[idx], *description.sample_block.getByPosition(idx).column);
}
else
{
bool is_nullable = description.types[idx].second;
if (is_nullable)
{
ColumnNullable & column_nullable = assert_cast<ColumnNullable &>(*columns[idx]);
insertValue(column_nullable.getNestedColumn(), description.types[idx].first, *value, name, array_info, idx);
column_nullable.getNullMapData().emplace_back(0);
}
else
insertValue(*columns[idx], description.types[idx].first, *value, name, array_info, idx);
}
}
}
if (cursor.cursorID() == 0)
{
all_read = true;
break;
}
}
if (num_rows == 0)
return {};
return Chunk(std::move(columns), num_rows);
}
}
#endif

92
src/Processors/Sources/MongoDBPocoLegacySource.h
View File

@ -1,92 +0,0 @@
#pragma once
#include "config.h"
#if USE_MONGODB
#include <Poco/MongoDB/Element.h>
#include <Poco/MongoDB/Array.h>
#include <Core/Block.h>
#include <Processors/ISource.h>
#include <Core/ExternalResultDescription.h>
#include <Core/Field.h>
namespace Poco
{
namespace MongoDB
{
class Connection;
class Document;
class Cursor;
class OpMsgCursor;
}
}
namespace DB
{
struct MongoDBPocoLegacyArrayInfo
{
size_t num_dimensions;
Field default_value;
std::function<Field(const Poco::MongoDB::Element & value, const std::string & name)> parser;
};
void authenticate(Poco::MongoDB::Connection & connection, const std::string & database, const std::string & user, const std::string & password);
bool isMongoDBWireProtocolOld(Poco::MongoDB::Connection & connection_, const std::string & database_name_);
/// Deprecated, will be removed soon.
class MongoDBPocoLegacyCursor
{
public:
MongoDBPocoLegacyCursor(
const std::string & database,
const std::string & collection,
const Block & sample_block_to_select,
const Poco::MongoDB::Document & query,
Poco::MongoDB::Connection & connection);
Poco::MongoDB::Document::Vector nextDocuments(Poco::MongoDB::Connection & connection);
Int64 cursorID() const;
private:
const bool is_wire_protocol_old;
std::unique_ptr<Poco::MongoDB::Cursor> old_cursor;
std::unique_ptr<Poco::MongoDB::OpMsgCursor> new_cursor;
Int64 cursor_id = 0;
};
/// Converts MongoDB Cursor to a stream of Blocks. Deprecated, will be removed soon.
class MongoDBPocoLegacySource final : public ISource
{
public:
MongoDBPocoLegacySource(
std::shared_ptr<Poco::MongoDB::Connection> & connection_,
const String & database_name_,
const String & collection_name_,
const Poco::MongoDB::Document & query_,
const Block & sample_block,
UInt64 max_block_size_);
~MongoDBPocoLegacySource() override;
String getName() const override { return "MongoDB"; }
private:
Chunk generate() override;
std::shared_ptr<Poco::MongoDB::Connection> connection;
MongoDBPocoLegacyCursor cursor;
const UInt64 max_block_size;
ExternalResultDescription description;
bool all_read = false;
std::unordered_map<size_t, MongoDBPocoLegacyArrayInfo> array_info;
};
}
#endif

327
src/Storages/StorageMongoDBPocoLegacy.cpp
View File

@ -1,327 +0,0 @@
#include "config.h"
#if USE_MONGODB
#include <Storages/StorageMongoDBPocoLegacy.h>
#include <Storages/StorageMongoDBPocoLegacySocketFactory.h>
#include <Storages/StorageFactory.h>
#include <Storages/checkAndGetLiteralArgument.h>
#include <Storages/NamedCollectionsHelpers.h>
#include <Poco/MongoDB/Connection.h>
#include <Poco/MongoDB/Cursor.h>
#include <Poco/MongoDB/Database.h>
#include <Poco/URI.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Core/Settings.h>
#include <Interpreters/Context.h>
#include <Common/parseAddress.h>
#include <Common/NamedCollections/NamedCollections.h>
#include <Common/RemoteHostFilter.h>
#include <IO/Operators.h>
#include <QueryPipeline/Pipe.h>
#include <Processors/Sources/MongoDBPocoLegacySource.h>
#include <base/range.h>
#include <unordered_set>
#include <Parsers/ASTLiteral.h>
#include <Processors/Sinks/SinkToStorage.h>
#include <DataTypes/DataTypeArray.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int MONGODB_CANNOT_AUTHENTICATE;
}
StorageMongoDBPocoLegacy::StorageMongoDBPocoLegacy(
const StorageID & table_id_,
const std::string & host_,
uint16_t port_,
const std::string & database_name_,
const std::string & collection_name_,
const std::string & username_,
const std::string & password_,
const std::string & options_,
const ColumnsDescription & columns_,
const ConstraintsDescription & constraints_,
const String & comment)
: IStorage(table_id_)
, database_name(database_name_)
, collection_name(collection_name_)
, username(username_)
, password(password_)
, uri("mongodb://" + host_ + ":" + std::to_string(port_) + "/" + database_name_ + "?" + options_)
{
LOG_WARNING(getLogger("StorageMongoDB (" + table_id_.table_name + ")"), "The deprecated MongoDB integartion implementation is used, this will be removed in next releases.");
StorageInMemoryMetadata storage_metadata;
storage_metadata.setColumns(columns_);
storage_metadata.setConstraints(constraints_);
storage_metadata.setComment(comment);
setInMemoryMetadata(storage_metadata);
}
void StorageMongoDBPocoLegacy::connectIfNotConnected()
{
std::lock_guard lock{connection_mutex};
if (!connection)
{
StorageMongoDBPocoLegacySocketFactory factory;
connection = std::make_shared<Poco::MongoDB::Connection>(uri, factory);
}
if (!authenticated)
{
Poco::URI poco_uri(uri);
auto query_params = poco_uri.getQueryParameters();
auto auth_source = std::find_if(query_params.begin(), query_params.end(),
[&](const std::pair<std::string, std::string> & param) { return param.first == "authSource"; });
auto auth_db = database_name;
if (auth_source != query_params.end())
auth_db = auth_source->second;
if (!username.empty() && !password.empty())
{
Poco::MongoDB::Database poco_db(auth_db);
if (!poco_db.authenticate(*connection, username, password, Poco::MongoDB::Database::AUTH_SCRAM_SHA1))
throw Exception(ErrorCodes::MONGODB_CANNOT_AUTHENTICATE, "Cannot authenticate in MongoDB, incorrect user or password");
}
authenticated = true;
}
}
class StorageMongoDBLegacySink : public SinkToStorage
{
public:
explicit StorageMongoDBLegacySink(
const std::string & collection_name_,
const std::string & db_name_,
const StorageMetadataPtr & metadata_snapshot_,
std::shared_ptr<Poco::MongoDB::Connection> connection_)
: SinkToStorage(metadata_snapshot_->getSampleBlock())
, collection_name(collection_name_)
, db_name(db_name_)
, metadata_snapshot{metadata_snapshot_}
, connection(connection_)
, is_wire_protocol_old(isMongoDBWireProtocolOld(*connection_, db_name))
{
}
String getName() const override { return "StorageMongoDBLegacySink"; }
void consume(Chunk & chunk) override
{
Poco::MongoDB::Database db(db_name);
Poco::MongoDB::Document::Vector documents;
auto block = getHeader().cloneWithColumns(chunk.getColumns());
size_t num_rows = block.rows();
size_t num_cols = block.columns();
const auto columns = block.getColumns();
const auto data_types = block.getDataTypes();
const auto data_names = block.getNames();
documents.reserve(num_rows);
for (const auto i : collections::range(0, num_rows))
{
Poco::MongoDB::Document::Ptr document = new Poco::MongoDB::Document();
for (const auto j : collections::range(0, num_cols))
{
insertValueIntoMongoDB(*document, data_names[j], *data_types[j], *columns[j], i);
}
documents.push_back(std::move(document));
}
if (is_wire_protocol_old)
{
Poco::SharedPtr<Poco::MongoDB::InsertRequest> insert_request = db.createInsertRequest(collection_name);
insert_request->documents() = std::move(documents);
connection->sendRequest(*insert_request);
}
else
{
Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> insert_request = db.createOpMsgMessage(collection_name);
insert_request->setCommandName(Poco::MongoDB::OpMsgMessage::CMD_INSERT);
insert_request->documents() = std::move(documents);
connection->sendRequest(*insert_request);
}
}
private:
void insertValueIntoMongoDB(
Poco::MongoDB::Document & document,
const std::string & name,
const IDataType & data_type,
const IColumn & column,
size_t idx)
{
WhichDataType which(data_type);
if (which.isArray())
{
const ColumnArray & column_array = assert_cast<const ColumnArray &>(column);
const ColumnArray::Offsets & offsets = column_array.getOffsets();
size_t offset = offsets[idx - 1];
size_t next_offset = offsets[idx];
const IColumn & nested_column = column_array.getData();
const auto * array_type = assert_cast<const DataTypeArray *>(&data_type);
const DataTypePtr & nested_type = array_type->getNestedType();
Poco::MongoDB::Array::Ptr array = new Poco::MongoDB::Array();
for (size_t i = 0; i + offset < next_offset; ++i)
{
insertValueIntoMongoDB(*array, Poco::NumberFormatter::format(i), *nested_type, nested_column, i + offset);
}
document.add(name, array);
return;
}
/// MongoDB does not support UInt64 type, so just cast it to Int64
if (which.isNativeUInt())
document.add(name, static_cast<Poco::Int64>(column.getUInt(idx)));
else if (which.isNativeInt())
document.add(name, static_cast<Poco::Int64>(column.getInt(idx)));
else if (which.isFloat32())
document.add(name, static_cast<Float64>(column.getFloat32(idx)));
else if (which.isFloat64())
document.add(name, column.getFloat64(idx));
else if (which.isDate())
document.add(name, Poco::Timestamp(DateLUT::instance().fromDayNum(DayNum(column.getUInt(idx))) * 1000000));
else if (which.isDateTime())
document.add(name, Poco::Timestamp(column.getUInt(idx) * 1000000));
else
{
WriteBufferFromOwnString ostr;
data_type.getDefaultSerialization()->serializeText(column, idx, ostr, FormatSettings{});
document.add(name, ostr.str());
}
}
String collection_name;
String db_name;
StorageMetadataPtr metadata_snapshot;
std::shared_ptr<Poco::MongoDB::Connection> connection;
const bool is_wire_protocol_old;
};
Pipe StorageMongoDBPocoLegacy::read(
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
SelectQueryInfo & /*query_info*/,
ContextPtr /*context*/,
QueryProcessingStage::Enum /*processed_stage*/,
size_t max_block_size,
size_t /*num_streams*/)
{
connectIfNotConnected();
storage_snapshot->check(column_names);
Block sample_block;
for (const String & column_name : column_names)
{
auto column_data = storage_snapshot->metadata->getColumns().getPhysical(column_name);
sample_block.insert({ column_data.type, column_data.name });
}
return Pipe(std::make_shared<MongoDBPocoLegacySource>(connection, database_name, collection_name, Poco::MongoDB::Document{}, sample_block, max_block_size));
}
SinkToStoragePtr StorageMongoDBPocoLegacy::write(const ASTPtr & /* query */, const StorageMetadataPtr & metadata_snapshot, ContextPtr /* context */, bool /*async_insert*/)
{
connectIfNotConnected();
return std::make_shared<StorageMongoDBLegacySink>(collection_name, database_name, metadata_snapshot, connection);
}
StorageMongoDBPocoLegacy::Configuration StorageMongoDBPocoLegacy::getConfiguration(ASTs engine_args, ContextPtr context)
{
Configuration configuration;
if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context))
{
validateNamedCollection(
*named_collection,
ValidateKeysMultiset<MongoDBEqualKeysSet>{"host", "port", "user", "username", "password", "database", "db", "collection", "table"},
{"options"});
configuration.host = named_collection->getAny<String>({"host", "hostname"});
configuration.port = static_cast<UInt16>(named_collection->get<UInt64>("port"));
configuration.username = named_collection->getAny<String>({"user", "username"});
configuration.password = named_collection->get<String>("password");
configuration.database = named_collection->getAny<String>({"database", "db"});
configuration.table = named_collection->getAny<String>({"collection", "table"});
configuration.options = named_collection->getOrDefault<String>("options", "");
}
else
{
if (engine_args.size() < 5 || engine_args.size() > 6)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Storage MongoDB requires from 5 to 6 parameters: "
"MongoDB('host:port', database, collection, 'user', 'password' [, 'options']).");
for (auto & engine_arg : engine_args)
engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context);
/// 27017 is the default MongoDB port.
auto parsed_host_port = parseAddress(checkAndGetLiteralArgument<String>(engine_args[0], "host:port"), 27017);
configuration.host = parsed_host_port.first;
configuration.port = parsed_host_port.second;
configuration.database = checkAndGetLiteralArgument<String>(engine_args[1], "database");
configuration.table = checkAndGetLiteralArgument<String>(engine_args[2], "table");
configuration.username = checkAndGetLiteralArgument<String>(engine_args[3], "username");
configuration.password = checkAndGetLiteralArgument<String>(engine_args[4], "password");
if (engine_args.size() >= 6)
configuration.options = checkAndGetLiteralArgument<String>(engine_args[5], "database");
}
context->getRemoteHostFilter().checkHostAndPort(configuration.host, toString(configuration.port));
return configuration;
}
void registerStorageMongoDBPocoLegacy(StorageFactory & factory)
{
factory.registerStorage("MongoDB", [](const StorageFactory::Arguments & args)
{
auto configuration = StorageMongoDBPocoLegacy::getConfiguration(args.engine_args, args.getLocalContext());
return std::make_shared<StorageMongoDBPocoLegacy>(
args.table_id,
configuration.host,
configuration.port,
configuration.database,
configuration.table,
configuration.username,
configuration.password,
configuration.options,
args.columns,
args.constraints,
args.comment);
},
{
.source_access_type = AccessType::MONGO,
});
}
}
#endif

79
src/Storages/StorageMongoDBPocoLegacy.h
View File

@ -1,79 +0,0 @@
#pragma once
#include "config.h"
#if USE_MONGODB
#include <Poco/MongoDB/Connection.h>
#include <Storages/IStorage.h>
namespace DB
{
/* Implements storage in the MongoDB database.
* Use ENGINE = MongoDB(host:port, database, collection, user, password [, options]);
* Read only.
*/
/// Deprecated, will be removed soon.
class StorageMongoDBPocoLegacy final : public IStorage
{
public:
StorageMongoDBPocoLegacy(
const StorageID & table_id_,
const std::string & host_,
uint16_t port_,
const std::string & database_name_,
const std::string & collection_name_,
const std::string & username_,
const std::string & password_,
const std::string & options_,
const ColumnsDescription & columns_,
const ConstraintsDescription & constraints_,
const String & comment);
std::string getName() const override { return "MongoDB"; }
Pipe read(
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
SelectQueryInfo & query_info,
ContextPtr context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
size_t num_streams) override;
SinkToStoragePtr write(
const ASTPtr & query,
const StorageMetadataPtr & /*metadata_snapshot*/,
ContextPtr context,
bool async_insert) override;
struct Configuration
{
std::string host;
UInt16 port;
std::string username;
std::string password;
std::string database;
std::string table;
std::string options;
};
static Configuration getConfiguration(ASTs engine_args, ContextPtr context);
private:
void connectIfNotConnected();
const std::string database_name;
const std::string collection_name;
const std::string username;
const std::string password;
const std::string uri;
std::shared_ptr<Poco::MongoDB::Connection> connection;
bool authenticated = false;
std::mutex connection_mutex; /// Protects the variables `connection` and `authenticated`.
};
}
#endif

57
src/Storages/StorageMongoDBPocoLegacySocketFactory.cpp
View File

@ -1,57 +0,0 @@
#include "config.h"
#if USE_MONGODB
#include "StorageMongoDBPocoLegacySocketFactory.h"
#include <Common/Exception.h>
#include <Poco/Net/IPAddress.h>
#include <Poco/Net/SocketAddress.h>
#if USE_SSL
# include <Poco/Net/SecureStreamSocket.h>
#endif
namespace DB
{
namespace ErrorCodes
{
extern const int FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME;
}
Poco::Net::StreamSocket StorageMongoDBPocoLegacySocketFactory::createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure)
{
return secure ? createSecureSocket(host, port, connectTimeout) : createPlainSocket(host, port, connectTimeout);
}
Poco::Net::StreamSocket StorageMongoDBPocoLegacySocketFactory::createPlainSocket(const std::string & host, int port, Poco::Timespan connectTimeout)
{
Poco::Net::SocketAddress address(host, port);
Poco::Net::StreamSocket socket;
socket.connect(address, connectTimeout);
return socket;
}
Poco::Net::StreamSocket StorageMongoDBPocoLegacySocketFactory::createSecureSocket(const std::string & host [[maybe_unused]], int port [[maybe_unused]], Poco::Timespan connectTimeout [[maybe_unused]])
{
#if USE_SSL
Poco::Net::SocketAddress address(host, port);
Poco::Net::SecureStreamSocket socket;
socket.setPeerHostName(host);
socket.connect(address, connectTimeout);
return socket;
#else
throw Exception(ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME, "SSL is not enabled at build time.");
#endif
}
}
#endif

24
src/Storages/StorageMongoDBPocoLegacySocketFactory.h
View File

@ -1,24 +0,0 @@
#pragma once
#include "config.h"
#if USE_MONGODB
#include <Poco/MongoDB/Connection.h>
namespace DB
{
/// Deprecated, will be removed soon.
class StorageMongoDBPocoLegacySocketFactory : public Poco::MongoDB::Connection::SocketFactory
{
public:
Poco::Net::StreamSocket createSocket(const std::string & host, int port, Poco::Timespan connectTimeout, bool secure) override;
private:
static Poco::Net::StreamSocket createPlainSocket(const std::string & host, int port, Poco::Timespan connectTimeout);
static Poco::Net::StreamSocket createSecureSocket(const std::string & host, int port, Poco::Timespan connectTimeout);
};
}
#endif

7
src/Storages/registerStorages.cpp
View File

@ -106,7 +106,7 @@ void registerStorageKeeperMap(StorageFactory & factory);
void registerStorageObjectStorage(StorageFactory & factory);
void registerStorages(bool use_legacy_mongodb_integration [[maybe_unused]])
void registerStorages()
{
auto & factory = StorageFactory::instance();
@ -169,10 +169,7 @@ void registerStorages(bool use_legacy_mongodb_integration [[maybe_unused]])
#endif
#if USE_MONGODB
if (use_legacy_mongodb_integration)
registerStorageMongoDBPocoLegacy(factory);
else
registerStorageMongoDB(factory);
registerStorageMongoDB(factory);
#endif
registerStorageRedis(factory);

2
src/Storages/registerStorages.h
View File

@ -2,5 +2,5 @@
namespace DB
{
void registerStorages(bool use_legacy_mongodb_integration);
void registerStorages();
}

128
src/TableFunctions/TableFunctionMongoDBPocoLegacy.cpp
View File

@ -1,128 +0,0 @@
#include "config.h"
#if USE_MONGODB
#include <Storages/StorageMongoDBPocoLegacy.h>
#include <Common/Exception.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Interpreters/parseColumnsListForTableFunction.h>
#include <TableFunctions/registerTableFunctions.h>
#include <Storages/checkAndGetLiteralArgument.h>
#include <Storages/ColumnsDescription.h>
#include <TableFunctions/TableFunctionMongoDB.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
namespace
{
/// Deprecated, will be removed soon.
class TableFunctionMongoDBPocoLegacy : public ITableFunction
{
public:
static constexpr auto name = "mongodb";
std::string getName() const override { return name; }
private:
StoragePtr executeImpl(
const ASTPtr & ast_function, ContextPtr context,
const std::string & table_name, ColumnsDescription cached_columns, bool is_insert_query) const override;
const char * getStorageTypeName() const override { return "MongoDB"; }
ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override;
void parseArguments(const ASTPtr & ast_function, ContextPtr context) override;
std::optional<StorageMongoDBPocoLegacy::Configuration> configuration;
String structure;
};
StoragePtr TableFunctionMongoDBPocoLegacy::executeImpl(const ASTPtr & /*ast_function*/,
ContextPtr context, const String & table_name, ColumnsDescription /*cached_columns*/, bool is_insert_query) const
{
auto columns = getActualTableStructure(context, is_insert_query);
auto storage = std::make_shared<StorageMongoDBPocoLegacy>(
StorageID(configuration->database, table_name),
configuration->host,
configuration->port,
configuration->database,
configuration->table,
configuration->username,
configuration->password,
configuration->options,
columns,
ConstraintsDescription(),
String{});
storage->startup();
return storage;
}
ColumnsDescription TableFunctionMongoDBPocoLegacy::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const
{
return parseColumnsListFromString(structure, context);
}
void TableFunctionMongoDBPocoLegacy::parseArguments(const ASTPtr & ast_function, ContextPtr context)
{
const auto & func_args = ast_function->as<ASTFunction &>();
if (!func_args.arguments)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function 'mongodb' must have arguments.");
ASTs & args = func_args.arguments->children;
if (args.size() < 6 || args.size() > 7)
{
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Table function 'mongodb' requires from 6 to 7 parameters: "
"mongodb('host:port', database, collection, 'user', 'password', structure, [, 'options'])");
}
ASTs main_arguments(args.begin(), args.begin() + 5);
for (size_t i = 5; i < args.size(); ++i)
{
if (const auto * ast_func = typeid_cast<const ASTFunction *>(args[i].get()))
{
const auto & [arg_name, arg_value] = getKeyValueMongoDBArgument(ast_func);
if (arg_name == "structure")
structure = checkAndGetLiteralArgument<String>(arg_value, "structure");
else if (arg_name == "options")
main_arguments.push_back(arg_value);
}
else if (i == 5)
{
structure = checkAndGetLiteralArgument<String>(args[i], "structure");
}
else if (i == 6)
{
main_arguments.push_back(args[i]);
}
}
configuration = StorageMongoDBPocoLegacy::getConfiguration(main_arguments, context);
}
}
void registerTableFunctionMongoDBPocoLegacy(TableFunctionFactory & factory)
{
factory.registerFunction<TableFunctionMongoDBPocoLegacy>();
}
}
#endif

7
src/TableFunctions/registerTableFunctions.cpp
View File

@ -3,7 +3,7 @@
namespace DB
{
void registerTableFunctions(bool use_legacy_mongodb_integration [[maybe_unused]])
void registerTableFunctions()
{
auto & factory = TableFunctionFactory::instance();
@ -23,10 +23,7 @@ void registerTableFunctions(bool use_legacy_mongodb_integration [[maybe_unused]]
registerTableFunctionInput(factory);
registerTableFunctionGenerate(factory);
#if USE_MONGODB
if (use_legacy_mongodb_integration)
registerTableFunctionMongoDBPocoLegacy(factory);
else
registerTableFunctionMongoDB(factory);
registerTableFunctionMongoDB(factory);
#endif
registerTableFunctionRedis(factory);
registerTableFunctionMergeTreeIndex(factory);

2
src/TableFunctions/registerTableFunctions.h
View File

@ -73,6 +73,6 @@ void registerDataLakeTableFunctions(TableFunctionFactory & factory);
void registerTableFunctionTimeSeries(TableFunctionFactory & factory);
void registerTableFunctions(bool use_legacy_mongodb_integration [[maybe_unused]]);
void registerTableFunctions();
}

10
tests/integration/helpers/external_sources.py
View File

@ -170,7 +170,6 @@ class SourceMongo(ExternalSource):
user,
password,
secure=False,
legacy=False,
):
ExternalSource.__init__(
self,
@ -183,13 +182,10 @@ class SourceMongo(ExternalSource):
password,
)
self.secure = secure
self.legacy = legacy
def get_source_str(self, table_name):
options = ""
if self.secure and self.legacy:
options = "<options>ssl=true</options>"
if self.secure and not self.legacy:
if self.secure:
options = "<options>tls=true&amp;tlsAllowInvalidCertificates=true</options>"
return """
@ -267,9 +263,7 @@ class SourceMongoURI(SourceMongo):
def get_source_str(self, table_name):
options = ""
if self.secure and self.legacy:
options = "ssl=true"
if self.secure and not self.legacy:
if self.secure:
options = "tls=true&amp;tlsAllowInvalidCertificates=true"
return """

3
tests/integration/test_dictionaries_all_layouts_separate_sources/configs/mongo/legacy.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<use_legacy_mongodb_integration>1</use_legacy_mongodb_integration>
</clickhouse>

3
tests/integration/test_dictionaries_all_layouts_separate_sources/configs/mongo/new.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<use_legacy_mongodb_integration>0</use_legacy_mongodb_integration>
</clickhouse>

37
tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py
View File

@ -19,20 +19,14 @@ def secure_connection(request):
@pytest.fixture(scope="module")
def legacy(request):
return request.param
@pytest.fixture(scope="module")
def cluster(secure_connection, legacy):
def cluster(secure_connection):
cluster_name = __file__.removeprefix("test_").removesuffix(".py")
cluster_name += "_legacy" if legacy else "_new"
cluster_name += "_secure" if secure_connection else "_insecure"
return ClickHouseCluster(cluster_name)
@pytest.fixture(scope="module")
def source(secure_connection, legacy, cluster):
def source(secure_connection, cluster):
return SourceMongo(
"MongoDB",
"localhost",
@ -42,7 +36,6 @@ def source(secure_connection, legacy, cluster):
"root",
"clickhouse",
secure=secure_connection,
legacy=legacy,
)
@ -69,18 +62,10 @@ def ranged_tester(source):
@pytest.fixture(scope="module")
def main_config(secure_connection, legacy):
if legacy:
main_config = [os.path.join("configs", "mongo", "legacy.xml")]
else:
main_config = [os.path.join("configs", "mongo", "new.xml")]
def main_config(secure_connection):
if secure_connection:
main_config.append(os.path.join("configs", "disable_ssl_verification.xml"))
else:
main_config.append(os.path.join("configs", "ssl_verification.xml"))
return main_config
return [os.path.join("configs", "disable_ssl_verification.xml")]
return [os.path.join("configs", "ssl_verification.xml")]
@pytest.fixture(scope="module")
@ -113,32 +98,28 @@ def started_cluster(
@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"])
@pytest.mark.parametrize("legacy", [False, True], indirect=["legacy"])
@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE))
def test_simple(secure_connection, legacy, started_cluster, layout_name, simple_tester):
def test_simple(secure_connection, started_cluster, layout_name, simple_tester):
simple_tester.execute(layout_name, started_cluster.instances["node1"])
@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"])
@pytest.mark.parametrize("legacy", [False, True], indirect=["legacy"])
@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX))
def test_complex(
secure_connection, legacy, started_cluster, layout_name, complex_tester
secure_connection, started_cluster, layout_name, complex_tester
):
complex_tester.execute(layout_name, started_cluster.instances["node1"])
@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"])
@pytest.mark.parametrize("legacy", [False, True], indirect=["legacy"])
@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED))
def test_ranged(secure_connection, legacy, started_cluster, layout_name, ranged_tester):
def test_ranged(secure_connection, started_cluster, layout_name, ranged_tester):
ranged_tester.execute(layout_name, started_cluster.instances["node1"])
@pytest.mark.parametrize("secure_connection", [True], indirect=["secure_connection"])
@pytest.mark.parametrize("legacy", [False, True], indirect=["legacy"])
@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE))
def test_simple_ssl(
secure_connection, legacy, started_cluster, layout_name, simple_tester
secure_connection, started_cluster, layout_name, simple_tester
):
simple_tester.execute(layout_name, started_cluster.instances["node1"])

30
tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py
View File

@ -17,18 +17,13 @@ def secure_connection(request):
return request.param
@pytest.fixture(scope="module")
def legacy(request):
return request.param
@pytest.fixture(scope="module")
def cluster(secure_connection):
return ClickHouseCluster(__file__)
@pytest.fixture(scope="module")
def source(secure_connection, legacy, cluster):
def source(secure_connection, cluster):
return SourceMongoURI(
"MongoDB",
"localhost",
@ -38,7 +33,6 @@ def source(secure_connection, legacy, cluster):
"root",
"clickhouse",
secure=secure_connection,
legacy=legacy,
)
@ -51,22 +45,14 @@ def simple_tester(source):
@pytest.fixture(scope="module")
def main_config(secure_connection, legacy):
if legacy:
main_config = [os.path.join("configs", "mongo", "legacy.xml")]
else:
main_config = [os.path.join("configs", "mongo", "new.xml")]
def main_config(secure_connection):
if secure_connection:
main_config.append(os.path.join("configs", "disable_ssl_verification.xml"))
else:
main_config.append(os.path.join("configs", "ssl_verification.xml"))
return main_config
return [os.path.join("configs", "disable_ssl_verification.xml")]
return [os.path.join("configs", "ssl_verification.xml")]
@pytest.fixture(scope="module")
def started_cluster(secure_connection, legacy, cluster, main_config, simple_tester):
def started_cluster(secure_connection, cluster, main_config, simple_tester):
dictionaries = simple_tester.list_dictionaries()
node = cluster.add_instance(
@ -85,16 +71,14 @@ def started_cluster(secure_connection, legacy, cluster, main_config, simple_test
# See comment in SourceMongoURI
@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"])
@pytest.mark.parametrize("legacy", [False, True], indirect=["legacy"])
@pytest.mark.parametrize("layout_name", ["flat"])
def test_simple(secure_connection, legacy, started_cluster, simple_tester, layout_name):
def test_simple(secure_connection, started_cluster, simple_tester, layout_name):
simple_tester.execute(layout_name, started_cluster.instances["uri_node"])
@pytest.mark.parametrize("secure_connection", [True], indirect=["secure_connection"])
@pytest.mark.parametrize("legacy", [False, True], indirect=["legacy"])
@pytest.mark.parametrize("layout_name", ["flat"])
def test_simple_ssl(
secure_connection, legacy, started_cluster, simple_tester, layout_name
secure_connection, started_cluster, simple_tester, layout_name
):
simple_tester.execute(layout_name, started_cluster.instances["uri_node"])

3
tests/integration/test_storage_mongodb/configs/feature_flag.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<use_legacy_mongodb_integration>0</use_legacy_mongodb_integration>
</clickhouse>

1
tests/integration/test_storage_mongodb/test.py
View File

@ -17,7 +17,6 @@ def started_cluster(request):
"node",
main_configs=[
"configs/named_collections.xml",
"configs/feature_flag.xml",
],
user_configs=["configs/users.xml"],
with_mongo=True,

0
tests/integration/test_storage_mongodb_legacy/__init__.py
View File

3
tests/integration/test_storage_mongodb_legacy/configs/feature_flag.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<use_legacy_mongodb_integration>1</use_legacy_mongodb_integration>
</clickhouse>

12
tests/integration/test_storage_mongodb_legacy/configs/named_collections.xml
View File

@ -1,12 +0,0 @@
<clickhouse>
<named_collections>
<mongo1>
<user>root</user>
<password>clickhouse</password>
<host>mongo1</host>
<port>27017</port>
<database>test</database>
<collection>simple_table</collection>
</mongo1>
</named_collections>
</clickhouse>

9
tests/integration/test_storage_mongodb_legacy/configs/users.xml
View File

@ -1,9 +0,0 @@
<clickhouse>
<users>
<default>
<password></password>
<profile>default</profile>
<named_collection_control>1</named_collection_control>
</default>
</users>
</clickhouse>

24
tests/integration/test_storage_mongodb_legacy/mongo_secure_config/cert.crt
View File

@ -1,24 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIEGzCCAwOgAwIBAgIUaoGlyuJAyvs6yowFXymfu7seEiUwDQYJKoZIhvcNAQEL
BQAwgZwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDbGlja0hvdXNlMRMwEQYDVQQH
DApDbGlja0hvdXNlMREwDwYDVQQKDAhQZXJzb25hbDETMBEGA1UECwwKQ2xpY2tI
b3VzZTEkMCIGCSqGSIb3DQEJARYVY2xpY2tob3VzZUBjbGlja2hvdXNlMRUwEwYD
VQQDDAxtb25nb19zZWN1cmUwHhcNMjQwNTI2MTYwMDMxWhcNMzQwNTI0MTYwMDMx
WjCBnDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNsaWNrSG91c2UxEzARBgNVBAcM
CkNsaWNrSG91c2UxETAPBgNVBAoMCFBlcnNvbmFsMRMwEQYDVQQLDApDbGlja0hv
dXNlMSQwIgYJKoZIhvcNAQkBFhVjbGlja2hvdXNlQGNsaWNraG91c2UxFTATBgNV
BAMMDG1vbmdvX3NlY3VyZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AJSeQfMG7xd0+kPehYlEsEw0Sm1DB05SXVCEzIX3DFD6XJrd8eeWwlzYaBatkcwj
+8yvqske02X/3YwpzJyFizAqJIoKql5c5Yii2xH1S9PFP0y+LoJre+eQziHyO33t
eeedeGNJ05Sm2ZAzjfMQ7Rdh6S+gdIO4Y102iQR5yr2aTrh7tu7XkNCjwKTqMMvz
SikP1Rft2J6ECim+MjYCCtH/4yXGeEJ5epU4t3y6Q23B2ZEhY+sqUdwgK9pu8oe4
mkZ1Qvwakc9Qg12owRSDjBBYrPvghXVpkJ2JkgKTrIAIz9tZ53eDVHNXbWMAotov
jEmRSoGIS1yzwmQ9PdxUwYcCAwEAAaNTMFEwHQYDVR0OBBYEFJyz3Kt5XBDg5cvI
0v1ioqejqX+CMB8GA1UdIwQYMBaAFJyz3Kt5XBDg5cvI0v1ioqejqX+CMA8GA1Ud
EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAHAQFA5VMYvaQFnKtKfHg9TF
qfJ4uM3YsGdgsgmGWgflD1S4Z290H6Q2QvyZAEceTrlJxqArlWlVp5DAU6EeXjEh
QMAgdkJHF1Hg2jsZKPtdkb88UtuzwAME357T8NtEJSHzNE5QqYwlVM71JkWpdqvA
UUdOJbWhhJfowIf4tMmL1DUuIy2qYpoP/tEBXEw9uwpmZqb7KELwT3lRyOMaGFN7
RHVwbvJWlHiu83QDNaWz6ijQkWl3tCN6TWcFD1qc1x8GpMzjbsAAYbCx7fbHM2LD
9kGSCiyv5K0MLNK5u67RtUFfPHtyD8RA0TtxIZ4PEN/eFANKS2/5NEi1ZuZ5/Pk=
-----END CERTIFICATE-----

8
tests/integration/test_storage_mongodb_legacy/mongo_secure_config/config.d/ssl_conf.xml
View File

@ -1,8 +0,0 @@
<clickhouse>
<openSSL>
<client>
<!-- For self-signed certificate -->
<verificationMode>none</verificationMode>
</client>
</openSSL>
</clickhouse>

52
tests/integration/test_storage_mongodb_legacy/mongo_secure_config/key.pem
View File

@ -1,52 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCUnkHzBu8XdPpD
3oWJRLBMNEptQwdOUl1QhMyF9wxQ+lya3fHnlsJc2GgWrZHMI/vMr6rJHtNl/92M
KcychYswKiSKCqpeXOWIotsR9UvTxT9Mvi6Ca3vnkM4h8jt97XnnnXhjSdOUptmQ
M43zEO0XYekvoHSDuGNdNokEecq9mk64e7bu15DQo8Ck6jDL80opD9UX7diehAop
vjI2AgrR/+MlxnhCeXqVOLd8ukNtwdmRIWPrKlHcICvabvKHuJpGdUL8GpHPUINd
qMEUg4wQWKz74IV1aZCdiZICk6yACM/bWed3g1RzV21jAKLaL4xJkUqBiEtcs8Jk
PT3cVMGHAgMBAAECggEAAul6qiHchB+uQMCWyC5xTeRqAXR3tAv4Tj4fGJjkXY4Z
OrAjr9Kp38EvX1amgvUWV3FT3NMevDf5xd9OdzAA0g0uJIF+mAhYFW48i1FnQcHQ
mOf0zmiZR7l8o7ROb3JvooXHxW+ba/qjGPVwC801gJvruehgbOCRxh9DTRp7sH5K
BmcddhULhKBEQjWUmYNEM3A2axpdi3g1aYKERRLn8J0DXcItTwbxuxbNcs3erl8W
3yyv/JKmqnWF5sNyX3wEWuQcDEZZy+W7Hn4KPMxyU+WA5el5nJ8kFlxhpInmajwu
8Ytn6IEyThyXutVomosVBuP16QORl2Nad0hnQO9toQKBgQDDgiehXr3k2wfVaVOD
PocW4leXausIU2XcCn6FxTG9vLUDMPANw0MxgenC2nrjaUU9J9UjdRYgMcFGWrl4
E27wEn5e0nZ/Y7F2cfhuOc9vNmZ+eHm2KQRyfAjIVL5Hpldqk2jXyCnLBNeWGHSw
kPQMU+FLqmrOFUvXlD2my+OSHwKBgQDCmgS9r+xFh4BCB9dY6eyQJF/jYmAQHs26
80WJ6gAhbUw1O71uDtS9/3PZVXwwNCOHrcc49BPrpJdxGPHGvd2Q5y+j5LDDbQSZ
aLTiCZ2B0RM5Bd2dXD8gEHN4WCX7pJ/o4kDi4zONBmp5mg/tFfer5z5IU/1P7Wak
1Mu0JIHzmQKBgDNaNoqeVgaMuYwGtFbez6DlJtiwzrdLIJAheYYte5k4vdruub8D
sNyKIRp7RJgDCJq9obBEiuE98GRIZDrz78nDMco6QcHIL87KtNRO/vtZMKa7gkyk
jXR8u9nS2H/9YyytN3amLsQSq4XTOqM+D7xFNAIp6w/ibB9d4quzFj1FAoGBAKTE
x/LcO897NWuzO/D6z+QUCGR87R15F3SNenmVedrTskz4ciH3yMW+v5ZrPSWLX/IH
f8GHWD6TM+780eoW5L1GIh5BCjHN4rEJ6O3iekxqfD4x6zzL2F8Lztk8uZxh/Uuw
FoSFHybvIcQoYAe8K+KPfzq6cqb0OY6i5n920dkxAoGAJkw6ADqsJfH3NR+bQfgF
oEA1KqriMxyEJm44Y7E80C+iF4iNALF+Er9TSnr4mDxX5e/dW9d1YeS9o0nOfkpF
MaBmJfxqo4QQJLPRaxYQ2Jhfn7irir4BroxeNXQgNNhgSuKIvkfRyGYwl7P0AT4v
8H8rkZGneMD3gLB5MfnRhGk=
-----END PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIIEGzCCAwOgAwIBAgIUaoGlyuJAyvs6yowFXymfu7seEiUwDQYJKoZIhvcNAQEL
BQAwgZwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDbGlja0hvdXNlMRMwEQYDVQQH
DApDbGlja0hvdXNlMREwDwYDVQQKDAhQZXJzb25hbDETMBEGA1UECwwKQ2xpY2tI
b3VzZTEkMCIGCSqGSIb3DQEJARYVY2xpY2tob3VzZUBjbGlja2hvdXNlMRUwEwYD
VQQDDAxtb25nb19zZWN1cmUwHhcNMjQwNTI2MTYwMDMxWhcNMzQwNTI0MTYwMDMx
WjCBnDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNsaWNrSG91c2UxEzARBgNVBAcM
CkNsaWNrSG91c2UxETAPBgNVBAoMCFBlcnNvbmFsMRMwEQYDVQQLDApDbGlja0hv
dXNlMSQwIgYJKoZIhvcNAQkBFhVjbGlja2hvdXNlQGNsaWNraG91c2UxFTATBgNV
BAMMDG1vbmdvX3NlY3VyZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AJSeQfMG7xd0+kPehYlEsEw0Sm1DB05SXVCEzIX3DFD6XJrd8eeWwlzYaBatkcwj
+8yvqske02X/3YwpzJyFizAqJIoKql5c5Yii2xH1S9PFP0y+LoJre+eQziHyO33t
eeedeGNJ05Sm2ZAzjfMQ7Rdh6S+gdIO4Y102iQR5yr2aTrh7tu7XkNCjwKTqMMvz
SikP1Rft2J6ECim+MjYCCtH/4yXGeEJ5epU4t3y6Q23B2ZEhY+sqUdwgK9pu8oe4
mkZ1Qvwakc9Qg12owRSDjBBYrPvghXVpkJ2JkgKTrIAIz9tZ53eDVHNXbWMAotov
jEmRSoGIS1yzwmQ9PdxUwYcCAwEAAaNTMFEwHQYDVR0OBBYEFJyz3Kt5XBDg5cvI
0v1ioqejqX+CMB8GA1UdIwQYMBaAFJyz3Kt5XBDg5cvI0v1ioqejqX+CMA8GA1Ud
EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAHAQFA5VMYvaQFnKtKfHg9TF
qfJ4uM3YsGdgsgmGWgflD1S4Z290H6Q2QvyZAEceTrlJxqArlWlVp5DAU6EeXjEh
QMAgdkJHF1Hg2jsZKPtdkb88UtuzwAME357T8NtEJSHzNE5QqYwlVM71JkWpdqvA
UUdOJbWhhJfowIf4tMmL1DUuIy2qYpoP/tEBXEw9uwpmZqb7KELwT3lRyOMaGFN7
RHVwbvJWlHiu83QDNaWz6ijQkWl3tCN6TWcFD1qc1x8GpMzjbsAAYbCx7fbHM2LD
9kGSCiyv5K0MLNK5u67RtUFfPHtyD8RA0TtxIZ4PEN/eFANKS2/5NEi1ZuZ5/Pk=
-----END CERTIFICATE-----

52
tests/integration/test_storage_mongodb_legacy/mongo_secure_config/mongo_cert.pem
View File

@ -1,52 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCUnkHzBu8XdPpD
3oWJRLBMNEptQwdOUl1QhMyF9wxQ+lya3fHnlsJc2GgWrZHMI/vMr6rJHtNl/92M
KcychYswKiSKCqpeXOWIotsR9UvTxT9Mvi6Ca3vnkM4h8jt97XnnnXhjSdOUptmQ
M43zEO0XYekvoHSDuGNdNokEecq9mk64e7bu15DQo8Ck6jDL80opD9UX7diehAop
vjI2AgrR/+MlxnhCeXqVOLd8ukNtwdmRIWPrKlHcICvabvKHuJpGdUL8GpHPUINd
qMEUg4wQWKz74IV1aZCdiZICk6yACM/bWed3g1RzV21jAKLaL4xJkUqBiEtcs8Jk
PT3cVMGHAgMBAAECggEAAul6qiHchB+uQMCWyC5xTeRqAXR3tAv4Tj4fGJjkXY4Z
OrAjr9Kp38EvX1amgvUWV3FT3NMevDf5xd9OdzAA0g0uJIF+mAhYFW48i1FnQcHQ
mOf0zmiZR7l8o7ROb3JvooXHxW+ba/qjGPVwC801gJvruehgbOCRxh9DTRp7sH5K
BmcddhULhKBEQjWUmYNEM3A2axpdi3g1aYKERRLn8J0DXcItTwbxuxbNcs3erl8W
3yyv/JKmqnWF5sNyX3wEWuQcDEZZy+W7Hn4KPMxyU+WA5el5nJ8kFlxhpInmajwu
8Ytn6IEyThyXutVomosVBuP16QORl2Nad0hnQO9toQKBgQDDgiehXr3k2wfVaVOD
PocW4leXausIU2XcCn6FxTG9vLUDMPANw0MxgenC2nrjaUU9J9UjdRYgMcFGWrl4
E27wEn5e0nZ/Y7F2cfhuOc9vNmZ+eHm2KQRyfAjIVL5Hpldqk2jXyCnLBNeWGHSw
kPQMU+FLqmrOFUvXlD2my+OSHwKBgQDCmgS9r+xFh4BCB9dY6eyQJF/jYmAQHs26
80WJ6gAhbUw1O71uDtS9/3PZVXwwNCOHrcc49BPrpJdxGPHGvd2Q5y+j5LDDbQSZ
aLTiCZ2B0RM5Bd2dXD8gEHN4WCX7pJ/o4kDi4zONBmp5mg/tFfer5z5IU/1P7Wak
1Mu0JIHzmQKBgDNaNoqeVgaMuYwGtFbez6DlJtiwzrdLIJAheYYte5k4vdruub8D
sNyKIRp7RJgDCJq9obBEiuE98GRIZDrz78nDMco6QcHIL87KtNRO/vtZMKa7gkyk
jXR8u9nS2H/9YyytN3amLsQSq4XTOqM+D7xFNAIp6w/ibB9d4quzFj1FAoGBAKTE
x/LcO897NWuzO/D6z+QUCGR87R15F3SNenmVedrTskz4ciH3yMW+v5ZrPSWLX/IH
f8GHWD6TM+780eoW5L1GIh5BCjHN4rEJ6O3iekxqfD4x6zzL2F8Lztk8uZxh/Uuw
FoSFHybvIcQoYAe8K+KPfzq6cqb0OY6i5n920dkxAoGAJkw6ADqsJfH3NR+bQfgF
oEA1KqriMxyEJm44Y7E80C+iF4iNALF+Er9TSnr4mDxX5e/dW9d1YeS9o0nOfkpF
MaBmJfxqo4QQJLPRaxYQ2Jhfn7irir4BroxeNXQgNNhgSuKIvkfRyGYwl7P0AT4v
8H8rkZGneMD3gLB5MfnRhGk=
-----END PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIIEGzCCAwOgAwIBAgIUaoGlyuJAyvs6yowFXymfu7seEiUwDQYJKoZIhvcNAQEL
BQAwgZwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDbGlja0hvdXNlMRMwEQYDVQQH
DApDbGlja0hvdXNlMREwDwYDVQQKDAhQZXJzb25hbDETMBEGA1UECwwKQ2xpY2tI
b3VzZTEkMCIGCSqGSIb3DQEJARYVY2xpY2tob3VzZUBjbGlja2hvdXNlMRUwEwYD
VQQDDAxtb25nb19zZWN1cmUwHhcNMjQwNTI2MTYwMDMxWhcNMzQwNTI0MTYwMDMx
WjCBnDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNsaWNrSG91c2UxEzARBgNVBAcM
CkNsaWNrSG91c2UxETAPBgNVBAoMCFBlcnNvbmFsMRMwEQYDVQQLDApDbGlja0hv
dXNlMSQwIgYJKoZIhvcNAQkBFhVjbGlja2hvdXNlQGNsaWNraG91c2UxFTATBgNV
BAMMDG1vbmdvX3NlY3VyZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AJSeQfMG7xd0+kPehYlEsEw0Sm1DB05SXVCEzIX3DFD6XJrd8eeWwlzYaBatkcwj
+8yvqske02X/3YwpzJyFizAqJIoKql5c5Yii2xH1S9PFP0y+LoJre+eQziHyO33t
eeedeGNJ05Sm2ZAzjfMQ7Rdh6S+gdIO4Y102iQR5yr2aTrh7tu7XkNCjwKTqMMvz
SikP1Rft2J6ECim+MjYCCtH/4yXGeEJ5epU4t3y6Q23B2ZEhY+sqUdwgK9pu8oe4
mkZ1Qvwakc9Qg12owRSDjBBYrPvghXVpkJ2JkgKTrIAIz9tZ53eDVHNXbWMAotov
jEmRSoGIS1yzwmQ9PdxUwYcCAwEAAaNTMFEwHQYDVR0OBBYEFJyz3Kt5XBDg5cvI
0v1ioqejqX+CMB8GA1UdIwQYMBaAFJyz3Kt5XBDg5cvI0v1ioqejqX+CMA8GA1Ud
EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAHAQFA5VMYvaQFnKtKfHg9TF
qfJ4uM3YsGdgsgmGWgflD1S4Z290H6Q2QvyZAEceTrlJxqArlWlVp5DAU6EeXjEh
QMAgdkJHF1Hg2jsZKPtdkb88UtuzwAME357T8NtEJSHzNE5QqYwlVM71JkWpdqvA
UUdOJbWhhJfowIf4tMmL1DUuIy2qYpoP/tEBXEw9uwpmZqb7KELwT3lRyOMaGFN7
RHVwbvJWlHiu83QDNaWz6ijQkWl3tCN6TWcFD1qc1x8GpMzjbsAAYbCx7fbHM2LD
9kGSCiyv5K0MLNK5u67RtUFfPHtyD8RA0TtxIZ4PEN/eFANKS2/5NEi1ZuZ5/Pk=
-----END CERTIFICATE-----

6
tests/integration/test_storage_mongodb_legacy/mongo_secure_config/mongo_secure.conf
View File

@ -1,6 +0,0 @@
net:
ssl:
mode: requireSSL
PEMKeyFile: /mongo/key.pem
CAFile: /mongo/cert.crt
allowConnectionsWithoutCertificates: true

509
tests/integration/test_storage_mongodb_legacy/test.py
View File

@ -1,509 +0,0 @@
import datetime
from uuid import UUID
import pymongo
import pytest
from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster
@pytest.fixture(scope="module")
def started_cluster(request):
try:
cluster = ClickHouseCluster(__file__)
node = cluster.add_instance(
"node",
main_configs=[
"mongo_secure_config/config.d/ssl_conf.xml",
"configs/named_collections.xml",
"configs/feature_flag.xml",
],
user_configs=["configs/users.xml"],
with_mongo=True,
)
cluster.start()
yield cluster
finally:
cluster.shutdown()
def get_mongo_connection(started_cluster, secure=False, with_credentials=True):
connection_str = ""
if with_credentials:
connection_str = "mongodb://root:clickhouse@localhost:{}".format(
started_cluster.mongo_secure_port if secure else started_cluster.mongo_port
)
else:
connection_str = "mongodb://localhost:{}".format(
started_cluster.mongo_no_cred_port
)
if secure:
connection_str += "/?tls=true&tlsAllowInvalidCertificates=true"
return pymongo.MongoClient(connection_str)
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_uuid(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
mongo_table = db["uuid_table"]
mongo_table.insert({"key": 0, "data": UUID("f0e77736-91d1-48ce-8f01-15123ca1c7ed")})
node = started_cluster.instances["node"]
node.query(
"CREATE TABLE uuid_mongo_table(key UInt64, data UUID) ENGINE = MongoDB('mongo1:27017', 'test', 'uuid_table', 'root', 'clickhouse')"
)
assert node.query("SELECT COUNT() FROM uuid_mongo_table") == "1\n"
assert (
node.query("SELECT data from uuid_mongo_table where key = 0")
== "f0e77736-91d1-48ce-8f01-15123ca1c7ed\n"
)
node.query("DROP TABLE uuid_mongo_table")
mongo_table.drop()
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_simple_select(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
simple_mongo_table = db["simple_table"]
data = []
for i in range(0, 100):
data.append({"key": i, "data": hex(i * i)})
simple_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query(
"CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')"
)
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
assert (
node.query("SELECT sum(key) FROM simple_mongo_table")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from simple_mongo_table where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE simple_mongo_table")
simple_mongo_table.drop()
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_simple_select_from_view(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
simple_mongo_table = db["simple_table"]
data = []
for i in range(0, 100):
data.append({"key": i, "data": hex(i * i)})
simple_mongo_table.insert_many(data)
simple_mongo_table_view = db.create_collection(
"simple_table_view", viewOn="simple_table"
)
node = started_cluster.instances["node"]
node.query(
"CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table_view', 'root', 'clickhouse')"
)
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
assert (
node.query("SELECT sum(key) FROM simple_mongo_table")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from simple_mongo_table where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE simple_mongo_table")
simple_mongo_table_view.drop()
simple_mongo_table.drop()
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_arrays(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
arrays_mongo_table = db["arrays_table"]
data = []
for i in range(0, 100):
data.append(
{
"key": i,
"arr_int64": [-(i + 1), -(i + 2), -(i + 3)],
"arr_int32": [-(i + 1), -(i + 2), -(i + 3)],
"arr_int16": [-(i + 1), -(i + 2), -(i + 3)],
"arr_int8": [-(i + 1), -(i + 2), -(i + 3)],
"arr_uint64": [i + 1, i + 2, i + 3],
"arr_uint32": [i + 1, i + 2, i + 3],
"arr_uint16": [i + 1, i + 2, i + 3],
"arr_uint8": [i + 1, i + 2, i + 3],
"arr_float32": [i + 1.125, i + 2.5, i + 3.750],
"arr_float64": [i + 1.125, i + 2.5, i + 3.750],
"arr_date": [
datetime.datetime(2002, 10, 27),
datetime.datetime(2024, 1, 8),
],
"arr_datetime": [
datetime.datetime(2023, 3, 31, 6, 3, 12),
datetime.datetime(1999, 2, 28, 12, 46, 34),
],
"arr_string": [str(i + 1), str(i + 2), str(i + 3)],
"arr_uuid": [
"f0e77736-91d1-48ce-8f01-15123ca1c7ed",
"93376a07-c044-4281-a76e-ad27cf6973c5",
],
"arr_mongo_uuid": [
UUID("f0e77736-91d1-48ce-8f01-15123ca1c7ed"),
UUID("93376a07-c044-4281-a76e-ad27cf6973c5"),
],
"arr_arr_bool": [
[True, False, True],
[True],
[],
None,
[False],
[None],
],
"arr_empty": [],
"arr_null": None,
"arr_nullable": None,
}
)
arrays_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query(
"CREATE TABLE arrays_mongo_table("
"key UInt64,"
"arr_int64 Array(Int64),"
"arr_int32 Array(Int32),"
"arr_int16 Array(Int16),"
"arr_int8 Array(Int8),"
"arr_uint64 Array(UInt64),"
"arr_uint32 Array(UInt32),"
"arr_uint16 Array(UInt16),"
"arr_uint8 Array(UInt8),"
"arr_float32 Array(Float32),"
"arr_float64 Array(Float64),"
"arr_date Array(Date),"
"arr_datetime Array(DateTime),"
"arr_string Array(String),"
"arr_uuid Array(UUID),"
"arr_mongo_uuid Array(UUID),"
"arr_arr_bool Array(Array(Bool)),"
"arr_empty Array(UInt64),"
"arr_null Array(UInt64),"
"arr_arr_null Array(Array(UInt64)),"
"arr_nullable Array(Nullable(UInt64))"
") ENGINE = MongoDB('mongo1:27017', 'test', 'arrays_table', 'root', 'clickhouse')"
)
assert node.query("SELECT COUNT() FROM arrays_mongo_table") == "100\n"
for column_name in ["arr_int64", "arr_int32", "arr_int16", "arr_int8"]:
assert (
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
== "[-43,-44,-45]\n"
)
for column_name in ["arr_uint64", "arr_uint32", "arr_uint16", "arr_uint8"]:
assert (
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
== "[43,44,45]\n"
)
for column_name in ["arr_float32", "arr_float64"]:
assert (
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
== "[43.125,44.5,45.75]\n"
)
assert (
node.query(f"SELECT arr_date FROM arrays_mongo_table WHERE key = 42")
== "['2002-10-27','2024-01-08']\n"
)
assert (
node.query(f"SELECT arr_datetime FROM arrays_mongo_table WHERE key = 42")
== "['2023-03-31 06:03:12','1999-02-28 12:46:34']\n"
)
assert (
node.query(f"SELECT arr_string FROM arrays_mongo_table WHERE key = 42")
== "['43','44','45']\n"
)
assert (
node.query(f"SELECT arr_uuid FROM arrays_mongo_table WHERE key = 42")
== "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
)
assert (
node.query(f"SELECT arr_mongo_uuid FROM arrays_mongo_table WHERE key = 42")
== "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
)
assert (
node.query(f"SELECT arr_arr_bool FROM arrays_mongo_table WHERE key = 42")
== "[[true,false,true],[true],[],[],[false],[false]]\n"
)
assert (
node.query(f"SELECT arr_empty FROM arrays_mongo_table WHERE key = 42") == "[]\n"
)
assert (
node.query(f"SELECT arr_null FROM arrays_mongo_table WHERE key = 42") == "[]\n"
)
assert (
node.query(f"SELECT arr_arr_null FROM arrays_mongo_table WHERE key = 42")
== "[]\n"
)
assert (
node.query(f"SELECT arr_nullable FROM arrays_mongo_table WHERE key = 42")
== "[]\n"
)
# Test INSERT SELECT
node.query("INSERT INTO arrays_mongo_table SELECT * FROM arrays_mongo_table")
assert node.query("SELECT COUNT() FROM arrays_mongo_table") == "200\n"
assert node.query("SELECT COUNT(DISTINCT *) FROM arrays_mongo_table") == "100\n"
node.query("DROP TABLE arrays_mongo_table")
arrays_mongo_table.drop()
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_complex_data_type(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
incomplete_mongo_table = db["complex_table"]
data = []
for i in range(0, 100):
data.append({"key": i, "data": hex(i * i), "dict": {"a": i, "b": str(i)}})
incomplete_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query(
"CREATE TABLE incomplete_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse')"
)
assert node.query("SELECT COUNT() FROM incomplete_mongo_table") == "100\n"
assert (
node.query("SELECT sum(key) FROM incomplete_mongo_table")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from incomplete_mongo_table where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE incomplete_mongo_table")
incomplete_mongo_table.drop()
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_incorrect_data_type(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
strange_mongo_table = db["strange_table"]
data = []
for i in range(0, 100):
data.append({"key": i, "data": hex(i * i), "aaaa": "Hello"})
strange_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query(
"CREATE TABLE strange_mongo_table(key String, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')"
)
with pytest.raises(QueryRuntimeException):
node.query("SELECT COUNT() FROM strange_mongo_table")
with pytest.raises(QueryRuntimeException):
node.query("SELECT uniq(key) FROM strange_mongo_table")
node.query(
"CREATE TABLE strange_mongo_table2(key UInt64, data String, bbbb String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')"
)
node.query("DROP TABLE strange_mongo_table")
node.query("DROP TABLE strange_mongo_table2")
strange_mongo_table.drop()
@pytest.mark.parametrize("started_cluster", [True], indirect=["started_cluster"])
def test_secure_connection(started_cluster):
mongo_connection = get_mongo_connection(started_cluster, secure=True)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
simple_mongo_table = db["simple_table"]
data = []
for i in range(0, 100):
data.append({"key": i, "data": hex(i * i)})
simple_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query(
"CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo_secure:27017', 'test', 'simple_table', 'root', 'clickhouse', 'ssl=true')"
)
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
assert (
node.query("SELECT sum(key) FROM simple_mongo_table")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from simple_mongo_table where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE simple_mongo_table")
simple_mongo_table.drop()
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_predefined_connection_configuration(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
simple_mongo_table = db["simple_table"]
data = []
for i in range(0, 100):
data.append({"key": i, "data": hex(i * i)})
simple_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query("drop table if exists simple_mongo_table")
node.query(
"create table simple_mongo_table(key UInt64, data String) engine = MongoDB(mongo1)"
)
assert node.query("SELECT count() FROM simple_mongo_table") == "100\n"
simple_mongo_table.drop()
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_no_credentials(started_cluster):
mongo_connection = get_mongo_connection(started_cluster, with_credentials=False)
db = mongo_connection["test"]
simple_mongo_table = db["simple_table"]
data = []
for i in range(0, 100):
data.append({"key": i, "data": hex(i * i)})
simple_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query(
f"create table simple_mongo_table_2(key UInt64, data String) engine = MongoDB('mongo_no_cred:27017', 'test', 'simple_table', '', '')"
)
assert node.query("SELECT count() FROM simple_mongo_table_2") == "100\n"
simple_mongo_table.drop()
node.query("DROP TABLE IF EXISTS simple_mongo_table_2")
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_auth_source(started_cluster):
mongo_connection = get_mongo_connection(started_cluster, with_credentials=False)
admin_db = mongo_connection["admin"]
admin_db.add_user(
"root",
"clickhouse",
roles=[{"role": "userAdminAnyDatabase", "db": "admin"}, "readWriteAnyDatabase"],
)
simple_mongo_table_admin = admin_db["simple_table"]
data = []
for i in range(0, 50):
data.append({"key": i, "data": hex(i * i)})
simple_mongo_table_admin.insert_many(data)
db = mongo_connection["test"]
simple_mongo_table = db["simple_table"]
data = []
for i in range(0, 100):
data.append({"key": i, "data": hex(i * i)})
simple_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query(
"create table simple_mongo_table_fail(key UInt64, data String) engine = MongoDB('mongo_no_cred:27017', 'test', 'simple_table', 'root', 'clickhouse')"
)
node.query_and_get_error("SELECT count() FROM simple_mongo_table_fail")
node.query(
"create table simple_mongo_table_ok(key UInt64, data String) engine = MongoDB('mongo_no_cred:27017', 'test', 'simple_table', 'root', 'clickhouse', 'authSource=admin')"
)
assert node.query("SELECT count() FROM simple_mongo_table_ok") == "100\n"
simple_mongo_table.drop()
simple_mongo_table_admin.drop()
node.query("DROP TABLE IF EXISTS simple_mongo_table_ok")
node.query("DROP TABLE IF EXISTS simple_mongo_table_fail")
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_missing_columns(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
simple_mongo_table = db["simple_table"]
data = []
for i in range(0, 10):
data.append({"key": i, "data": hex(i * i)})
for i in range(0, 10):
data.append({"key": i})
simple_mongo_table.insert_many(data)
node = started_cluster.instances["node"]
node.query("drop table if exists simple_mongo_table")
node.query(
"create table simple_mongo_table(key UInt64, data Nullable(String)) engine = MongoDB(mongo1)"
)
result = node.query("SELECT count() FROM simple_mongo_table WHERE isNull(data)")
assert result == "10\n"
simple_mongo_table.drop()
node.query("DROP TABLE IF EXISTS simple_mongo_table")
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
def test_simple_insert_select(started_cluster):
mongo_connection = get_mongo_connection(started_cluster)
db = mongo_connection["test"]
db.add_user("root", "clickhouse")
simple_mongo_table = db["simple_table"]
node = started_cluster.instances["node"]
node.query("DROP TABLE IF EXISTS simple_mongo_table")
node.query(
"CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')"
)
node.query(
"INSERT INTO simple_mongo_table SELECT number, 'kek' || toString(number) FROM numbers(10)"
)
assert (
node.query("SELECT data from simple_mongo_table where key = 7").strip()
== "kek7"
)
node.query("INSERT INTO simple_mongo_table(key) SELECT 12")
assert int(node.query("SELECT count() from simple_mongo_table")) == 11
assert (
node.query("SELECT data from simple_mongo_table where key = 12").strip() == ""
)
node.query("DROP TABLE simple_mongo_table")
simple_mongo_table.drop()

3
tests/integration/test_table_function_mongodb/configs/feature_flag.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<use_legacy_mongodb_integration>0</use_legacy_mongodb_integration>
</clickhouse>

1
tests/integration/test_table_function_mongodb/test.py
View File

@ -14,7 +14,6 @@ def started_cluster(request):
with_mongo=True,
main_configs=[
"configs/named_collections.xml",
"configs/feature_flag.xml",
],
user_configs=["configs/users.xml"],
)

0
tests/integration/test_table_function_mongodb_legacy/__init__.py
View File

3
tests/integration/test_table_function_mongodb_legacy/configs/feature_flag.xml
View File

@ -1,3 +0,0 @@
<clickhouse>
<use_legacy_mongodb_integration>1</use_legacy_mongodb_integration>
</clickhouse>

9
tests/integration/test_table_function_mongodb_legacy/configs/users.xml
View File

@ -1,9 +0,0 @@
<clickhouse>
<users>
<default>
<password></password>
<profile>default</profile>
<named_collection_control>1</named_collection_control>
</default>
</users>
</clickhouse>

24
tests/integration/test_table_function_mongodb_legacy/mongo_secure_config/cert.crt
View File

@ -1,24 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIEGzCCAwOgAwIBAgIUaoGlyuJAyvs6yowFXymfu7seEiUwDQYJKoZIhvcNAQEL
BQAwgZwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDbGlja0hvdXNlMRMwEQYDVQQH
DApDbGlja0hvdXNlMREwDwYDVQQKDAhQZXJzb25hbDETMBEGA1UECwwKQ2xpY2tI
b3VzZTEkMCIGCSqGSIb3DQEJARYVY2xpY2tob3VzZUBjbGlja2hvdXNlMRUwEwYD
VQQDDAxtb25nb19zZWN1cmUwHhcNMjQwNTI2MTYwMDMxWhcNMzQwNTI0MTYwMDMx
WjCBnDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNsaWNrSG91c2UxEzARBgNVBAcM
CkNsaWNrSG91c2UxETAPBgNVBAoMCFBlcnNvbmFsMRMwEQYDVQQLDApDbGlja0hv
dXNlMSQwIgYJKoZIhvcNAQkBFhVjbGlja2hvdXNlQGNsaWNraG91c2UxFTATBgNV
BAMMDG1vbmdvX3NlY3VyZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AJSeQfMG7xd0+kPehYlEsEw0Sm1DB05SXVCEzIX3DFD6XJrd8eeWwlzYaBatkcwj
+8yvqske02X/3YwpzJyFizAqJIoKql5c5Yii2xH1S9PFP0y+LoJre+eQziHyO33t
eeedeGNJ05Sm2ZAzjfMQ7Rdh6S+gdIO4Y102iQR5yr2aTrh7tu7XkNCjwKTqMMvz
SikP1Rft2J6ECim+MjYCCtH/4yXGeEJ5epU4t3y6Q23B2ZEhY+sqUdwgK9pu8oe4
mkZ1Qvwakc9Qg12owRSDjBBYrPvghXVpkJ2JkgKTrIAIz9tZ53eDVHNXbWMAotov
jEmRSoGIS1yzwmQ9PdxUwYcCAwEAAaNTMFEwHQYDVR0OBBYEFJyz3Kt5XBDg5cvI
0v1ioqejqX+CMB8GA1UdIwQYMBaAFJyz3Kt5XBDg5cvI0v1ioqejqX+CMA8GA1Ud
EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAHAQFA5VMYvaQFnKtKfHg9TF
qfJ4uM3YsGdgsgmGWgflD1S4Z290H6Q2QvyZAEceTrlJxqArlWlVp5DAU6EeXjEh
QMAgdkJHF1Hg2jsZKPtdkb88UtuzwAME357T8NtEJSHzNE5QqYwlVM71JkWpdqvA
UUdOJbWhhJfowIf4tMmL1DUuIy2qYpoP/tEBXEw9uwpmZqb7KELwT3lRyOMaGFN7
RHVwbvJWlHiu83QDNaWz6ijQkWl3tCN6TWcFD1qc1x8GpMzjbsAAYbCx7fbHM2LD
9kGSCiyv5K0MLNK5u67RtUFfPHtyD8RA0TtxIZ4PEN/eFANKS2/5NEi1ZuZ5/Pk=
-----END CERTIFICATE-----

8
tests/integration/test_table_function_mongodb_legacy/mongo_secure_config/config.d/ssl_conf.xml
View File

@ -1,8 +0,0 @@
<clickhouse>
<openSSL>
<client>
<!-- For self-signed certificate -->
<verificationMode>none</verificationMode>
</client>
</openSSL>
</clickhouse>

Some files were not shown because too many files have changed in this diff Show More