Use a table to make InternalName::make more efficient for Python interned strings

dtool/src/interrogate/interfaceMakerPythonNative.cxx

@@ -2742,7 +2742,9 @@ int GetParnetDepth(CPPType *type) {
   } else if (TypeManager::is_wchar_pointer(type)) {
   } else if (TypeManager::is_pointer_to_PyObject(type)) {
   } else if (TypeManager::is_pointer_to_Py_buffer(type)) {
-  } else if (TypeManager::is_pointer(type) || TypeManager::is_reference(type) || TypeManager::is_struct(type)) {
+  } else if (TypeManager::is_pointer(type) ||
+             TypeManager::is_reference(type) ||
+             TypeManager::is_struct(type)) {
     ++answer;
     int deepest = 0;
     TypeIndex type_index = builder.get_type(TypeManager::unwrap(TypeManager::resolve_type(type)), false);
@@ -3275,6 +3277,21 @@ write_function_instance(ostream &out, InterfaceMaker::Object *obj,
       only_pyobjects = false;
       ++num_params;
 
+    } else if (TypeManager::is_pointer_to_PyStringObject(type)) {
+      if (args_type == AT_single_arg) {
+        // This is a single-arg function, so there's no need
+        // to convert anything.
+        param_name = "arg";
+        extra_param_check += " && PyString_Check(arg)";
+      } else {
+        indent(out, indent_level) << "PyStringObject *" << param_name << ";\n";
+        format_specifiers += "S";
+        parameter_list += ", &" + param_name;
+      }
+      pexpr_string = param_name;
+      expected_params += "string";
+      ++num_params;
+
     } else if (TypeManager::is_pointer_to_PyObject(type)) {
       if (args_type == AT_single_arg) {
         // This is a single-arg function, so there's no need
@@ -3604,7 +3621,7 @@ write_function_instance(ostream &out, InterfaceMaker::Object *obj,
   if (true) {
     indent(out, indent_level)
       << "if (PyErr_Occurred()) {\n";
-    delete_return_value(out, indent_level, remap, return_expr);
+    delete_return_value(out, indent_level + 2, remap, return_expr);
     indent(out, indent_level)
       << "  if (PyErr_ExceptionMatches(PyExc_TypeError)) {\n";
     indent(out, indent_level)

dtool/src/interrogate/typeManager.cxx

@@ -1199,8 +1199,52 @@ is_PyObject(CPPType *type) {
     return is_PyObject(type->as_const_type()->_wrapped_around);
 
   case CPPDeclaration::ST_extension:
+  case CPPDeclaration::ST_struct:
     return (type->get_local_name(&parser) == "PyObject" ||
-            type->get_local_name(&parser) == "_object");
+            type->get_local_name(&parser) == "PyTypeObject" ||
+            type->get_local_name(&parser) == "PyStringObject" ||
+            type->get_local_name(&parser) == "PyUnicodeObject" ||
+            type->get_local_name(&parser) == "_object" ||
+            type->get_local_name(&parser) == "_typeobject");
+
+  default:
+    return false;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: TypeManager::is_pointer_to_PyStringObject
+//       Access: Public, Static
+//  Description: Returns true if the indicated type is PyStringObject *.
+////////////////////////////////////////////////////////////////////
+bool TypeManager::
+is_pointer_to_PyStringObject(CPPType *type) {
+  switch (type->get_subtype()) {
+  case CPPDeclaration::ST_const:
+    return is_pointer_to_PyStringObject(type->as_const_type()->_wrapped_around);
+
+  case CPPDeclaration::ST_pointer:
+    return is_PyStringObject(type->as_pointer_type()->_pointing_at);
+
+  default:
+    return false;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: TypeManager::is_PyStringObject
+//       Access: Public, Static
+//  Description: Returns true if the indicated type is PyStringObject.
+////////////////////////////////////////////////////////////////////
+bool TypeManager::
+is_PyStringObject(CPPType *type) {
+  switch (type->get_subtype()) {
+  case CPPDeclaration::ST_const:
+    return is_PyStringObject(type->as_const_type()->_wrapped_around);
+
+  case CPPDeclaration::ST_extension:
+  case CPPDeclaration::ST_struct:
+    return (type->get_local_name(&parser) == "PyStringObject");
 
   default:
     return false;
@@ -1238,6 +1282,7 @@ is_Py_buffer(CPPType *type) {
     return is_Py_buffer(type->as_const_type()->_wrapped_around);
 
   case CPPDeclaration::ST_extension:
+  case CPPDeclaration::ST_struct:
     return (type->get_local_name(&parser) == "Py_buffer");
 
   default:

dtool/src/interrogate/typeManager.h

@@ -94,6 +94,8 @@ public:
   static bool is_const_ref_to_pointer_to_base(CPPType *type);
   static bool is_pointer_to_PyObject(CPPType *type);
   static bool is_PyObject(CPPType *type);
+  static bool is_pointer_to_PyStringObject(CPPType *type);
+  static bool is_PyStringObject(CPPType *type);
   static bool is_pointer_to_Py_buffer(CPPType *type);
   static bool is_Py_buffer(CPPType *type);
   static bool involves_unpublished(CPPType *type);

dtool/src/parser-inc/Python.h

@@ -23,6 +23,12 @@
 struct _object;
 typedef _object PyObject;
 
+struct _typeobject;
+typedef _typeobject PyTypeObject;
+
+struct PyStringObject;
+struct PyUnicodeObject;
+
 class PyThreadState;
 typedef int Py_ssize_t;
 struct Py_buffer;

dtool/src/pystub/pystub.cxx

@@ -121,6 +121,8 @@ extern "C" {
   EXPCL_PYSTUB int PyString_AsStringAndSize(...);
   EXPCL_PYSTUB int PyString_FromString(...);
   EXPCL_PYSTUB int PyString_FromStringAndSize(...);
+  EXPCL_PYSTUB int PyString_InternFromString(...);
+  EXPCL_PYSTUB int PyString_InternInPlace(...);
   EXPCL_PYSTUB int PyString_Size(...);
   EXPCL_PYSTUB int PyString_Type(...);
   EXPCL_PYSTUB int PySys_GetObject(...);
@@ -152,6 +154,8 @@ extern "C" {
   EXPCL_PYSTUB int PyUnicode_FromStringAndSize(...);
   EXPCL_PYSTUB int PyUnicode_FromWideChar(...);
   EXPCL_PYSTUB int PyUnicode_GetSize(...);
+  EXPCL_PYSTUB int PyUnicode_InternFromString(...);
+  EXPCL_PYSTUB int PyUnicode_InternInPlace(...);
   EXPCL_PYSTUB int PyUnicode_Type(...);
   EXPCL_PYSTUB int Py_BuildValue(...);
   EXPCL_PYSTUB int Py_InitModule4(...);
@@ -296,6 +300,8 @@ int PyString_AsString(...) { return 0; }
 int PyString_AsStringAndSize(...) { return 0; }
 int PyString_FromString(...) { return 0; }
 int PyString_FromStringAndSize(...) { return 0; }
+int PyString_InternFromString(...) { return 0; }
+int PyString_InternInPlace(...) { return 0; }
 int PyString_Size(...) { return 0; }
 int PyString_Type(...) { return 0; }
 int PySys_GetObject(...) { return 0; }
@@ -327,6 +333,8 @@ int PyUnicode_FromString(...) { return 0; }
 int PyUnicode_FromStringAndSize(...) { return 0; }
 int PyUnicode_FromWideChar(...) { return 0; }
 int PyUnicode_GetSize(...) { return 0; }
+int PyUnicode_InternFromString(...) { return 0; }
+int PyUnicode_InternInPlace(...) { return 0; }
 int PyUnicode_Type(...) { return 0; }
 int Py_BuildValue(...) { return 0; }
 int Py_InitModule4(...) { return 0; }

makepanda/makepanda.py

@@ -3168,6 +3168,7 @@ if (not RUNTIME):
   TargetAdd('libp3gobj.in', opts=['IMOD:panda3d.core', 'ILIB:libp3gobj', 'SRCDIR:panda/src/gobj'])
   TargetAdd('libp3gobj_igate.obj', input='libp3gobj.in', opts=["DEPENDENCYONLY"])
   TargetAdd('p3gobj_geomVertexArrayData_ext.obj', opts=OPTS, input='geomVertexArrayData_ext.cxx')
+  TargetAdd('p3gobj_internalName_ext.obj', opts=OPTS, input='internalName_ext.cxx')
 
 #
 # DIRECTORY: panda/src/pgraphnodes/
@@ -3539,6 +3540,7 @@ if (not RUNTIME):
   TargetAdd('libpanda.dll', input='p3putil_typedWritable_ext.obj')
   TargetAdd('libpanda.dll', input='p3pnmimage_pfmFile_ext.obj')
   TargetAdd('libpanda.dll', input='p3gobj_geomVertexArrayData_ext.obj')
+  TargetAdd('libpanda.dll', input='p3gobj_internalName_ext.obj')
   TargetAdd('libpanda.dll', input='p3pgraph_ext_composite.obj')
   TargetAdd('libpanda.dll', input='p3display_graphicsStateGuardian_ext.obj')
 

panda/src/gobj/Sources.pp

@@ -43,6 +43,7 @@
     geomVertexWriter.h geomVertexWriter.I \
     indexBufferContext.I indexBufferContext.h \
     internalName.I internalName.h \
+    internalName_ext.h internalName_ext.cxx \
     lens.h lens.I \
     material.I material.h materialPool.I materialPool.h  \
     matrixLens.I matrixLens.h \

panda/src/gobj/internalName.cxx

@@ -43,6 +43,10 @@ PT(InternalName) InternalName::_view;
 TypeHandle InternalName::_type_handle;
 TypeHandle InternalName::_texcoord_type_handle;
 
+#ifdef HAVE_PYTHON
+InternalName::PyInternTable InternalName::_py_intern_table;
+#endif
+
 ////////////////////////////////////////////////////////////////////
 //     Function: InternalName::Constructor
 //       Access: Private

panda/src/gobj/internalName.h

@@ -43,11 +43,13 @@ class EXPCL_PANDA_GOBJ InternalName : public TypedWritableReferenceCount {
 private:
   InternalName(InternalName *parent, const string &basename);
 
+public:
+  INLINE static PT(InternalName) make(const string &name);
+
 PUBLISHED:
   virtual ~InternalName();
   virtual bool unref() const;
 
-  INLINE static PT(InternalName) make(const string &name);
   static PT(InternalName) make(const string &name, int index);
   PT(InternalName) append(const string &basename);
 
@@ -88,6 +90,22 @@ PUBLISHED:
   INLINE static PT(InternalName) get_model();
   INLINE static PT(InternalName) get_view();
 
+#ifdef HAVE_PYTHON
+#if PY_MAJOR_VERSION >= 3
+  EXTENSION(static PT(InternalName) make(PyUnicodeObject *str));
+#else
+  EXTENSION(static PT(InternalName) make(PyStringObject *str));
+#endif
+#endif
+
+public:
+#ifdef HAVE_PYTHON
+  // It's OK for us to define it here since these are just pointers of
+  // which the reference is maintained indefinitely.
+  typedef phash_map<PyObject *, InternalName *, pointer_hash> PyInternTable;
+  static PyInternTable _py_intern_table;
+#endif
+
 private:
   PT(InternalName) _parent;
   string _basename;
@@ -157,5 +175,3 @@ INLINE ostream &operator << (ostream &out, const InternalName &tcn);
 
 #endif
 
-
-  

panda/src/gobj/internalName_ext.cxx

@@ -0,0 +1,82 @@
+// Filename: internalName_ext.I
+// Created by:  rdb (28Sep14)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#include "internalName_ext.h"
+
+#ifdef HAVE_PYTHON
+
+////////////////////////////////////////////////////////////////////
+//     Function: InternalName::make
+//       Access: Published, Static
+//  Description: This extension method serves to allow coercion of
+//               Python interned strings to InternalName objects
+//               more efficiently by storing a mapping between
+//               Python and Panda interned strings.
+////////////////////////////////////////////////////////////////////
+#if PY_MAJOR_VERSION >= 3
+PT(InternalName) Extension<InternalName>::
+make(PyUnicodeObject *str) {
+  if (!PyUnicode_CHECK_INTERNED(str)) {
+    // Not an interned string; don't bother.
+    Py_ssize_t len = 0;
+    char *c_str = PyUnicode_AsUTF8AndSize(str, &len);
+    return InternalName::make(name);
+  }
+
+  InternalName::PyInternTable::const_iterator it;
+  it = InternalName::_py_intern_table.find((PyObject*)str);
+
+  if (it != InternalName::_py_intern_table.end()) {
+    return (*it).second;
+
+  } else {
+    Py_ssize_t len = 0;
+    char *c_str = PyUnicode_AsUTF8AndSize(str, &len);
+    string name(c_str, len);
+
+#else
+PT(InternalName) Extension<InternalName>::
+make(PyStringObject *str) {
+  if (!PyString_CHECK_INTERNED(str)) {
+    // Not an interned string; don't bother.
+    string name(PyString_AS_STRING(str), PyString_GET_SIZE(str));
+    return InternalName::make(name);
+  }
+
+  InternalName::PyInternTable::const_iterator it;
+  it = InternalName::_py_intern_table.find((PyObject*)str);
+
+  if (it != InternalName::_py_intern_table.end()) {
+    return (*it).second;
+
+  } else {
+    string name(PyString_AS_STRING(str), PyString_GET_SIZE(str));
+
+#endif  // PY_MAJOR_VERSION
+
+    PT(InternalName) iname = InternalName::make(name);
+
+    // We basically leak references to both the PyObject and the
+    // InternalName.  We may want to change that in the future if it
+    // becomes a problem.
+    Py_INCREF(str);
+    iname->ref();
+
+    InternalName::_py_intern_table.insert(make_pair((PyObject *)str, iname.p()));
+    return iname.p();
+  }
+
+}
+
+#endif  // HAVE_PYTHON

panda/src/gobj/internalName_ext.h

@@ -0,0 +1,44 @@
+// Filename: internalName_ext.h
+// Created by:  rdb (28Sep14)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef INTERNALNAME_EXT_H
+#define INTERNALNAME_EXT_H
+
+#include "dtoolbase.h"
+
+#ifdef HAVE_PYTHON
+
+#include "extension.h"
+#include "internalName.h"
+#include "py_panda.h"
+
+////////////////////////////////////////////////////////////////////
+//       Class : Extension<InternalName>
+// Description : This class defines the extension methods for
+//               InternalName, which are called instead of
+//               any C++ methods with the same prototype.
+////////////////////////////////////////////////////////////////////
+template<>
+class Extension<InternalName> : public ExtensionBase<InternalName> {
+public:
+#if PY_MAJOR_VERSION >= 3
+  static PT(InternalName) make(PyUnicodeObject *str);
+#else
+  static PT(InternalName) make(PyStringObject *str);
+#endif
+};
+
+#endif  // HAVE_PYTHON
+
+#endif  // INTERNALNAME_EXT_H