From 24386cdc1eb78ef708fd060f69b673a7edd742ce Mon Sep 17 00:00:00 2001 From: rdb Date: Thu, 2 Oct 2014 18:28:06 +0000 Subject: [PATCH] Use a table to make InternalName::make more efficient for Python interned strings --- .../interfaceMakerPythonNative.cxx | 21 ++++- dtool/src/interrogate/typeManager.cxx | 47 ++++++++++- dtool/src/interrogate/typeManager.h | 2 + dtool/src/parser-inc/Python.h | 6 ++ dtool/src/pystub/pystub.cxx | 8 ++ makepanda/makepanda.py | 2 + panda/src/gobj/Sources.pp | 1 + panda/src/gobj/internalName.cxx | 4 + panda/src/gobj/internalName.h | 24 +++++- panda/src/gobj/internalName_ext.cxx | 82 +++++++++++++++++++ panda/src/gobj/internalName_ext.h | 44 ++++++++++ 11 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 panda/src/gobj/internalName_ext.cxx create mode 100644 panda/src/gobj/internalName_ext.h diff --git a/dtool/src/interrogate/interfaceMakerPythonNative.cxx b/dtool/src/interrogate/interfaceMakerPythonNative.cxx index 931eb6cc47..0a1f601c12 100755 --- a/dtool/src/interrogate/interfaceMakerPythonNative.cxx +++ b/dtool/src/interrogate/interfaceMakerPythonNative.cxx @@ -2742,7 +2742,9 @@ int GetParnetDepth(CPPType *type) { } else if (TypeManager::is_wchar_pointer(type)) { } else if (TypeManager::is_pointer_to_PyObject(type)) { } else if (TypeManager::is_pointer_to_Py_buffer(type)) { - } else if (TypeManager::is_pointer(type) || TypeManager::is_reference(type) || TypeManager::is_struct(type)) { + } else if (TypeManager::is_pointer(type) || + TypeManager::is_reference(type) || + TypeManager::is_struct(type)) { ++answer; int deepest = 0; TypeIndex type_index = builder.get_type(TypeManager::unwrap(TypeManager::resolve_type(type)), false); @@ -3275,6 +3277,21 @@ write_function_instance(ostream &out, InterfaceMaker::Object *obj, only_pyobjects = false; ++num_params; + } else if (TypeManager::is_pointer_to_PyStringObject(type)) { + if (args_type == AT_single_arg) { + // This is a single-arg function, so there's no need + // to convert anything. + param_name = "arg"; + extra_param_check += " && PyString_Check(arg)"; + } else { + indent(out, indent_level) << "PyStringObject *" << param_name << ";\n"; + format_specifiers += "S"; + parameter_list += ", &" + param_name; + } + pexpr_string = param_name; + expected_params += "string"; + ++num_params; + } else if (TypeManager::is_pointer_to_PyObject(type)) { if (args_type == AT_single_arg) { // This is a single-arg function, so there's no need @@ -3604,7 +3621,7 @@ write_function_instance(ostream &out, InterfaceMaker::Object *obj, if (true) { indent(out, indent_level) << "if (PyErr_Occurred()) {\n"; - delete_return_value(out, indent_level, remap, return_expr); + delete_return_value(out, indent_level + 2, remap, return_expr); indent(out, indent_level) << " if (PyErr_ExceptionMatches(PyExc_TypeError)) {\n"; indent(out, indent_level) diff --git a/dtool/src/interrogate/typeManager.cxx b/dtool/src/interrogate/typeManager.cxx index 245724b6ef..9ae0903469 100644 --- a/dtool/src/interrogate/typeManager.cxx +++ b/dtool/src/interrogate/typeManager.cxx @@ -1199,8 +1199,52 @@ is_PyObject(CPPType *type) { return is_PyObject(type->as_const_type()->_wrapped_around); case CPPDeclaration::ST_extension: + case CPPDeclaration::ST_struct: return (type->get_local_name(&parser) == "PyObject" || - type->get_local_name(&parser) == "_object"); + type->get_local_name(&parser) == "PyTypeObject" || + type->get_local_name(&parser) == "PyStringObject" || + type->get_local_name(&parser) == "PyUnicodeObject" || + type->get_local_name(&parser) == "_object" || + type->get_local_name(&parser) == "_typeobject"); + + default: + return false; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: TypeManager::is_pointer_to_PyStringObject +// Access: Public, Static +// Description: Returns true if the indicated type is PyStringObject *. +//////////////////////////////////////////////////////////////////// +bool TypeManager:: +is_pointer_to_PyStringObject(CPPType *type) { + switch (type->get_subtype()) { + case CPPDeclaration::ST_const: + return is_pointer_to_PyStringObject(type->as_const_type()->_wrapped_around); + + case CPPDeclaration::ST_pointer: + return is_PyStringObject(type->as_pointer_type()->_pointing_at); + + default: + return false; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: TypeManager::is_PyStringObject +// Access: Public, Static +// Description: Returns true if the indicated type is PyStringObject. +//////////////////////////////////////////////////////////////////// +bool TypeManager:: +is_PyStringObject(CPPType *type) { + switch (type->get_subtype()) { + case CPPDeclaration::ST_const: + return is_PyStringObject(type->as_const_type()->_wrapped_around); + + case CPPDeclaration::ST_extension: + case CPPDeclaration::ST_struct: + return (type->get_local_name(&parser) == "PyStringObject"); default: return false; @@ -1238,6 +1282,7 @@ is_Py_buffer(CPPType *type) { return is_Py_buffer(type->as_const_type()->_wrapped_around); case CPPDeclaration::ST_extension: + case CPPDeclaration::ST_struct: return (type->get_local_name(&parser) == "Py_buffer"); default: diff --git a/dtool/src/interrogate/typeManager.h b/dtool/src/interrogate/typeManager.h index 9123da52ed..afe02fa812 100644 --- a/dtool/src/interrogate/typeManager.h +++ b/dtool/src/interrogate/typeManager.h @@ -94,6 +94,8 @@ public: static bool is_const_ref_to_pointer_to_base(CPPType *type); static bool is_pointer_to_PyObject(CPPType *type); static bool is_PyObject(CPPType *type); + static bool is_pointer_to_PyStringObject(CPPType *type); + static bool is_PyStringObject(CPPType *type); static bool is_pointer_to_Py_buffer(CPPType *type); static bool is_Py_buffer(CPPType *type); static bool involves_unpublished(CPPType *type); diff --git a/dtool/src/parser-inc/Python.h b/dtool/src/parser-inc/Python.h index a059adca2c..329c8ed5bd 100755 --- a/dtool/src/parser-inc/Python.h +++ b/dtool/src/parser-inc/Python.h @@ -23,6 +23,12 @@ struct _object; typedef _object PyObject; +struct _typeobject; +typedef _typeobject PyTypeObject; + +struct PyStringObject; +struct PyUnicodeObject; + class PyThreadState; typedef int Py_ssize_t; struct Py_buffer; diff --git a/dtool/src/pystub/pystub.cxx b/dtool/src/pystub/pystub.cxx index 5fde787a8c..1553819636 100644 --- a/dtool/src/pystub/pystub.cxx +++ b/dtool/src/pystub/pystub.cxx @@ -121,6 +121,8 @@ extern "C" { EXPCL_PYSTUB int PyString_AsStringAndSize(...); EXPCL_PYSTUB int PyString_FromString(...); EXPCL_PYSTUB int PyString_FromStringAndSize(...); + EXPCL_PYSTUB int PyString_InternFromString(...); + EXPCL_PYSTUB int PyString_InternInPlace(...); EXPCL_PYSTUB int PyString_Size(...); EXPCL_PYSTUB int PyString_Type(...); EXPCL_PYSTUB int PySys_GetObject(...); @@ -152,6 +154,8 @@ extern "C" { EXPCL_PYSTUB int PyUnicode_FromStringAndSize(...); EXPCL_PYSTUB int PyUnicode_FromWideChar(...); EXPCL_PYSTUB int PyUnicode_GetSize(...); + EXPCL_PYSTUB int PyUnicode_InternFromString(...); + EXPCL_PYSTUB int PyUnicode_InternInPlace(...); EXPCL_PYSTUB int PyUnicode_Type(...); EXPCL_PYSTUB int Py_BuildValue(...); EXPCL_PYSTUB int Py_InitModule4(...); @@ -296,6 +300,8 @@ int PyString_AsString(...) { return 0; } int PyString_AsStringAndSize(...) { return 0; } int PyString_FromString(...) { return 0; } int PyString_FromStringAndSize(...) { return 0; } +int PyString_InternFromString(...) { return 0; } +int PyString_InternInPlace(...) { return 0; } int PyString_Size(...) { return 0; } int PyString_Type(...) { return 0; } int PySys_GetObject(...) { return 0; } @@ -327,6 +333,8 @@ int PyUnicode_FromString(...) { return 0; } int PyUnicode_FromStringAndSize(...) { return 0; } int PyUnicode_FromWideChar(...) { return 0; } int PyUnicode_GetSize(...) { return 0; } +int PyUnicode_InternFromString(...) { return 0; } +int PyUnicode_InternInPlace(...) { return 0; } int PyUnicode_Type(...) { return 0; } int Py_BuildValue(...) { return 0; } int Py_InitModule4(...) { return 0; } diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index c66a5670d1..0e654c2e7f 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -3168,6 +3168,7 @@ if (not RUNTIME): TargetAdd('libp3gobj.in', opts=['IMOD:panda3d.core', 'ILIB:libp3gobj', 'SRCDIR:panda/src/gobj']) TargetAdd('libp3gobj_igate.obj', input='libp3gobj.in', opts=["DEPENDENCYONLY"]) TargetAdd('p3gobj_geomVertexArrayData_ext.obj', opts=OPTS, input='geomVertexArrayData_ext.cxx') + TargetAdd('p3gobj_internalName_ext.obj', opts=OPTS, input='internalName_ext.cxx') # # DIRECTORY: panda/src/pgraphnodes/ @@ -3539,6 +3540,7 @@ if (not RUNTIME): TargetAdd('libpanda.dll', input='p3putil_typedWritable_ext.obj') TargetAdd('libpanda.dll', input='p3pnmimage_pfmFile_ext.obj') TargetAdd('libpanda.dll', input='p3gobj_geomVertexArrayData_ext.obj') + TargetAdd('libpanda.dll', input='p3gobj_internalName_ext.obj') TargetAdd('libpanda.dll', input='p3pgraph_ext_composite.obj') TargetAdd('libpanda.dll', input='p3display_graphicsStateGuardian_ext.obj') diff --git a/panda/src/gobj/Sources.pp b/panda/src/gobj/Sources.pp index a79670dd6d..8b71bfac5a 100644 --- a/panda/src/gobj/Sources.pp +++ b/panda/src/gobj/Sources.pp @@ -43,6 +43,7 @@ geomVertexWriter.h geomVertexWriter.I \ indexBufferContext.I indexBufferContext.h \ internalName.I internalName.h \ + internalName_ext.h internalName_ext.cxx \ lens.h lens.I \ material.I material.h materialPool.I materialPool.h \ matrixLens.I matrixLens.h \ diff --git a/panda/src/gobj/internalName.cxx b/panda/src/gobj/internalName.cxx index 8ee046338f..856fdcd403 100644 --- a/panda/src/gobj/internalName.cxx +++ b/panda/src/gobj/internalName.cxx @@ -43,6 +43,10 @@ PT(InternalName) InternalName::_view; TypeHandle InternalName::_type_handle; TypeHandle InternalName::_texcoord_type_handle; +#ifdef HAVE_PYTHON +InternalName::PyInternTable InternalName::_py_intern_table; +#endif + //////////////////////////////////////////////////////////////////// // Function: InternalName::Constructor // Access: Private diff --git a/panda/src/gobj/internalName.h b/panda/src/gobj/internalName.h index 7c3c42e259..82c570dcbb 100644 --- a/panda/src/gobj/internalName.h +++ b/panda/src/gobj/internalName.h @@ -43,11 +43,13 @@ class EXPCL_PANDA_GOBJ InternalName : public TypedWritableReferenceCount { private: InternalName(InternalName *parent, const string &basename); +public: + INLINE static PT(InternalName) make(const string &name); + PUBLISHED: virtual ~InternalName(); virtual bool unref() const; - INLINE static PT(InternalName) make(const string &name); static PT(InternalName) make(const string &name, int index); PT(InternalName) append(const string &basename); @@ -88,6 +90,22 @@ PUBLISHED: INLINE static PT(InternalName) get_model(); INLINE static PT(InternalName) get_view(); +#ifdef HAVE_PYTHON +#if PY_MAJOR_VERSION >= 3 + EXTENSION(static PT(InternalName) make(PyUnicodeObject *str)); +#else + EXTENSION(static PT(InternalName) make(PyStringObject *str)); +#endif +#endif + +public: +#ifdef HAVE_PYTHON + // It's OK for us to define it here since these are just pointers of + // which the reference is maintained indefinitely. + typedef phash_map PyInternTable; + static PyInternTable _py_intern_table; +#endif + private: PT(InternalName) _parent; string _basename; @@ -116,7 +134,7 @@ private: static PT(InternalName) _camera; static PT(InternalName) _model; static PT(InternalName) _view; - + public: // Datagram stuff static void register_with_read_factory(); @@ -157,5 +175,3 @@ INLINE ostream &operator << (ostream &out, const InternalName &tcn); #endif - - diff --git a/panda/src/gobj/internalName_ext.cxx b/panda/src/gobj/internalName_ext.cxx new file mode 100644 index 0000000000..99bee102a9 --- /dev/null +++ b/panda/src/gobj/internalName_ext.cxx @@ -0,0 +1,82 @@ +// Filename: internalName_ext.I +// Created by: rdb (28Sep14) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + +#include "internalName_ext.h" + +#ifdef HAVE_PYTHON + +//////////////////////////////////////////////////////////////////// +// Function: InternalName::make +// Access: Published, Static +// Description: This extension method serves to allow coercion of +// Python interned strings to InternalName objects +// more efficiently by storing a mapping between +// Python and Panda interned strings. +//////////////////////////////////////////////////////////////////// +#if PY_MAJOR_VERSION >= 3 +PT(InternalName) Extension:: +make(PyUnicodeObject *str) { + if (!PyUnicode_CHECK_INTERNED(str)) { + // Not an interned string; don't bother. + Py_ssize_t len = 0; + char *c_str = PyUnicode_AsUTF8AndSize(str, &len); + return InternalName::make(name); + } + + InternalName::PyInternTable::const_iterator it; + it = InternalName::_py_intern_table.find((PyObject*)str); + + if (it != InternalName::_py_intern_table.end()) { + return (*it).second; + + } else { + Py_ssize_t len = 0; + char *c_str = PyUnicode_AsUTF8AndSize(str, &len); + string name(c_str, len); + +#else +PT(InternalName) Extension:: +make(PyStringObject *str) { + if (!PyString_CHECK_INTERNED(str)) { + // Not an interned string; don't bother. + string name(PyString_AS_STRING(str), PyString_GET_SIZE(str)); + return InternalName::make(name); + } + + InternalName::PyInternTable::const_iterator it; + it = InternalName::_py_intern_table.find((PyObject*)str); + + if (it != InternalName::_py_intern_table.end()) { + return (*it).second; + + } else { + string name(PyString_AS_STRING(str), PyString_GET_SIZE(str)); + +#endif // PY_MAJOR_VERSION + + PT(InternalName) iname = InternalName::make(name); + + // We basically leak references to both the PyObject and the + // InternalName. We may want to change that in the future if it + // becomes a problem. + Py_INCREF(str); + iname->ref(); + + InternalName::_py_intern_table.insert(make_pair((PyObject *)str, iname.p())); + return iname.p(); + } + +} + +#endif // HAVE_PYTHON diff --git a/panda/src/gobj/internalName_ext.h b/panda/src/gobj/internalName_ext.h new file mode 100644 index 0000000000..83b1510428 --- /dev/null +++ b/panda/src/gobj/internalName_ext.h @@ -0,0 +1,44 @@ +// Filename: internalName_ext.h +// Created by: rdb (28Sep14) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + +#ifndef INTERNALNAME_EXT_H +#define INTERNALNAME_EXT_H + +#include "dtoolbase.h" + +#ifdef HAVE_PYTHON + +#include "extension.h" +#include "internalName.h" +#include "py_panda.h" + +//////////////////////////////////////////////////////////////////// +// Class : Extension +// Description : This class defines the extension methods for +// InternalName, which are called instead of +// any C++ methods with the same prototype. +//////////////////////////////////////////////////////////////////// +template<> +class Extension : public ExtensionBase { +public: +#if PY_MAJOR_VERSION >= 3 + static PT(InternalName) make(PyUnicodeObject *str); +#else + static PT(InternalName) make(PyStringObject *str); +#endif +}; + +#endif // HAVE_PYTHON + +#endif // INTERNALNAME_EXT_H