Tutorial

Understanding and expanding Python: hands-on experience with the Python internals

Cristián
maureira.dev

@cmaureir

The Structure

of this tutorial

🧑‍🏫 The tutorial has two parts
🖵 Each part will have a short presentation
🏃‍➡️ A set of exercises follows each presentation
⭐ Some exercises have optional follow-ups
⭐ If you finish quickly, continue with the next one
⭐ If you do everything even faster, help others!

First Part Presentation

Navigating cpython

CPython is the standard implementation

To get around it

We need to understand C

a little bit

Python and C (1/3)


    # A comment
    import my_module

    def add(a: int, b: float) -> float:
        return a + b


    def main():
        msg: str = "Hello World"
        x: int = 3
        y: float = 0.14
        z: float = add(x, y)
        print("%f" % z)

    if __name__ == "__main__":
        main()


    // A comment
    #include <my_module.h>

    float add(int a, float b) {
        return a + b;
    }

    int main(){
        char msg[] = "Hello World";
        int x = 3;
        float y = 0.14;
        float z = add(x, y);
        printf("%f", z);

        return 0;
    }

Python and C (2/3)


    l = [1, 2, 3, 4, 5]

    l2 = []

    l2.append(42)
    l2.append(17)
    l2.append("Hallo")


    int a[] = {1, 2, 3, 4, 5};

    int *a2 = malloc(3 * sizeof(int));

    a2[0] = 42;
    a2[1] = 17;
    a2[2] = "Hallo"; // BOOM! 🔥


    // Don't forget to free
    // the memory you allocated
    free(a2);

Python and C (3/3)


    typedef struct {
      size_t size;
      int *data;
    } int_vector;

    int *create_vector(size_t n) {
        return malloc(n * sizeof(int));
    }

    void resize_vector(int_vector *v, size_t n) {
      v->size = n + 1;
      v->data = realloc(v->data, v->size * sizeof(int));
    }

    void set_vector(int_vector *v, size_t n, int x) {
      if(v) {
        if(n >= v->size) {
          resize_vector(v, n);
        }
        v->data[n] = x;
      }
    }


    // Yay, we got "kind of" a vector
    int_vector *v = create_vector(10);
    set_vector(v, 0, 123);

We might need to reinvent the wheel

source

*

`int *a;`

C pointers (1/2)

The `cpython` repo structure

github.com/python/cpython

Doc - Official documentation
Include - Interpreter header files
Lib - stdlib in pure Python
Modules - stdlib in C
Objects - built-in types
Python - CPython runtime

PyObject (1/2)


typedef struct _object {
    _PyObject_HEAD_EXTRA
    Py_ssize_t ob_refcnt;
    struct _typeobject *ob_type;
} PyObject;

PyObject (2/2)

Using the GIL


struct _object {
    _Py_ANONYMOUS union {
#if SIZEOF_VOID_P > 4
        int64_t ob_refcnt_full; /* This field is needed
                                   for efficient initialization
                                   with Clang on ARM */
        struct {
#  if PY_BIG_ENDIAN
            uint16_t ob_flags;
            uint16_t ob_overflow;
            uint32_t ob_refcnt;
#  else
            uint32_t ob_refcnt;
            uint16_t ob_overflow;
            uint16_t ob_flags;
#  endif
        };
#else
        Py_ssize_t ob_refcnt;  // part of stable ABI; do not change
#endif
        _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, char) _aligner;
    };

    PyTypeObject *ob_type;  // part of stable ABI; do not change
};

Not using the GIL


struct _object {
    // ob_tid stores the thread id (or zero). It is also used
    // by the GC and the trashcan mechanism as a linked list
    // pointer and by the GC to store the computed "gc_refs"
    // refcount.
    _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, uintptr_t) ob_tid;
    uint16_t ob_flags;
    PyMutex ob_mutex;           // per-object lock
    uint8_t ob_gc_bits;         // gc-related state
    uint32_t ob_ref_local;      // local reference count
    Py_ssize_t ob_ref_shared;   // shared (atomic) reference count
    PyTypeObject *ob_type;
};


typedef struct _typeobject {
    PyObject_VAR_HEAD
    const char *tp_name; /* For printing, in format "." */
    Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */

    /* Methods to implement standard operations */

    destructor tp_dealloc;
    Py_ssize_t tp_vectorcall_offset;
    getattrfunc tp_getattr;
    setattrfunc tp_setattr;
    PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
                                    or tp_reserved (Python 3) */
    reprfunc tp_repr;

    /* Method suites for standard classes */

    PyNumberMethods *tp_as_number;
    PySequenceMethods *tp_as_sequence;
    PyMappingMethods *tp_as_mapping;

    /* More standard operations (here for binary compatibility) */

    hashfunc tp_hash;
    ternaryfunc tp_call;
    reprfunc tp_str;
    getattrofunc tp_getattro;
    setattrofunc tp_setattro;

    /* Functions to access object as input/output buffer */
    PyBufferProcs *tp_as_buffer;

    /* Flags to define presence of optional/expanded features */
    unsigned long tp_flags;

    const char *tp_doc; /* Documentation string */

    /* Assigned meaning in release 2.0 */
    /* call function for all accessible objects */
    traverseproc tp_traverse;

    /* delete references to contained objects */
    inquiry tp_clear;

    /* Assigned meaning in release 2.1 */
    /* rich comparisons */
    richcmpfunc tp_richcompare;

    /* weak reference enabler */
    Py_ssize_t tp_weaklistoffset;

    /* Iterators */
    getiterfunc tp_iter;
    iternextfunc tp_iternext;


    /* Attribute descriptor and subclassing stuff */
    struct PyMethodDef *tp_methods;
    struct PyMemberDef *tp_members;
    struct PyGetSetDef *tp_getset;
    struct _typeobject *tp_base;
    PyObject *tp_dict;
    descrgetfunc tp_descr_get;
    descrsetfunc tp_descr_set;
    Py_ssize_t tp_dictoffset;
    initproc tp_init;
    allocfunc tp_alloc;
    newfunc tp_new;
    freefunc tp_free; /* Low-level free-memory routine */
    inquiry tp_is_gc; /* For PyObject_IS_GC */
    PyObject *tp_bases;
    PyObject *tp_mro; /* method resolution order */
    PyObject *tp_cache;
    PyObject *tp_subclasses;
    PyObject *tp_weaklist;
    destructor tp_del;

    /* Type attribute cache version tag. Added in version 2.6 */
    unsigned int tp_version_tag;

    destructor tp_finalize;
    vectorcallfunc tp_vectorcall;

    /* bpo-37250: kept for backwards compatibility in CPython 3.8 only */
    Py_DEPRECATED(3.8) int (*tp_print)(PyObject *, FILE *, int);

#ifdef COUNT_ALLOCS
    /* these must be last and never explicitly initialized */
    Py_ssize_t tp_allocs;
    Py_ssize_t tp_frees;
    Py_ssize_t tp_maxalloc;
    struct _typeobject *tp_prev;
    struct _typeobject *tp_next;
#endif
} PyTypeObject;

/* The *real* layout of a type object when allocated on the heap */
typedef struct _heaptypeobject {
    /* Note: there's a dependency on the order of these members
       in slotptr() in typeobject.c . */
    PyTypeObject ht_type;
    PyAsyncMethods as_async;
    PyNumberMethods as_number;
    PyMappingMethods as_mapping;
    PySequenceMethods as_sequence; /* as_sequence comes after as_mapping,
                                      so that the mapping wins when both
                                      the mapping and the sequence define
                                      a given operator (e.g. __getitem__).
                                      see add_operators() in typeobject.c . */
    PyBufferProcs as_buffer;
    PyObject *ht_name, *ht_slots, *ht_qualname;
    struct _dictkeysobject *ht_cached_keys;
    /* here are optional user slots, followed by the members. */
} PyHeapTypeObject;

Python C-API

Strings

docs.python.org/3/c-api/unicode.html


  char msg[] = "Hello World!";
  PyObject *s = PyUnicode_FromString(msg);
  ...
  Py_ssize_t len = PyUnicode_GetLength(s);

Lists

docs.python.org/3/c-api/list.html


  char msg[] = "Hello World!";
  PyObject *s = PyUnicode_FromString(msg);

  PyObject *l = PyList_New(0);
  PyList_Append(l, s);
  ...

Check what's the PyObject storing


  int PyUnicode_Check(PyObject *o);

  int PyList_Check(PyObject *o);

  int PyDict_Check(PyObject *o);

  int PyFloat_Check(PyObject *o);

  int PyTuple_Check(PyObject *o);

Parse arguments


  static PyObject* glob(PyObject* self, PyObject* args,
                        PyObject* kwds){

    const char *kwlist[] = {"directory", "recursive", 0};
    const char *directory = nullptr;
    int recursive = 0;

    if (!PyArg_ParseTupleAndKeywords(args, kwds,
                                     "z|p",
                                     const_cast<char **>(kwlist),
                                     &directory,
                                     &recursive)) {
      return nullptr;
    }
    ...

How does CPython works?

From Python to Machine Code

Bytecode

➡️

Specialized
Bytecode

➡️

Micro-Op
traces

➡️

Optimized
Micro-Op
traces

Brandt Bucher: Building a JIT compiler for CPython (PyCon US 2024)

End of First Part Presentation

Navigating cpython

git clone https://github.com/cmaureir/cpython_tutorial

Compiler Setup

Linux: gcc
macOS: clang (via XCode)
Windows: cl (via MSVC2022)

Environment Setup

Terminal (command-line)
Preferable Linux or macOS
Optional: configure your favorite IDE

Exercise 00

Compile a `"Hello, World!"` example in C

Exercise 01

Build `cpython`

Exercise 01 - Build cpython

More info: devguide.python.org/getting-started/setup-building/

For Linux/macOS


    cd cpython
    ./configure --with-pydebug
    make -j16  # or more
    ./python

For Windows


    cd cpython
    PCBuild\build.bat
    PCbuild\amd64\python_d.exe

Exercise 02

Add a Python function to `cpython`

Exercise 02 - New Python function in cpython

Pick your favorite Python module (Lib/)
Add a new function (maybe based on an existing one)

Change parameters defaults
Remove parameters
Add new functionality

Recompile (?) cpython and try it out!

Exercise 03

Add a new built-in function to `cpython`

Exercise 03 - New C function in cpython

Consider functionality you know, and change it
For example:
- pdir() like dir() but for non_dunder attributes.

Exercise 04

Add a new method to a Python container

Exercise 05

Add an alias to a Python keyword

Second Part Presentation

Extending cpython

Creating a Python C-extension

docs.python.org/3/extending/extending.html

Parts of a C extension (1/4)


          static PyObject* hello(PyObject* self, PyObject* args) {
              char *msg = "Hey there!";
              return Py_BuildValue("s", msg);
          }

          static PyMethodDef the_methods[] = {
              {"hello", (PyCFunction)hello, METH_NOARGS, NULL},
              {NULL, NULL, 0, NULL}
          };

Parts of a C extension (2/4)


          static struct PyModuleDef module = {
              .m_base = PyModuleDef_HEAD_INIT,
              .m_name = "mymodule",
              .m_size = 0,
              .m_methods = the_methods,
          };

Parts of a C extension (3/4)


          PyMODINIT_FUNC PyInit_mymodule(void){
              return PyModuleDef_Init(&module);
          }

Parts of a C extension (4/4)


        >>> from mymodule import hello
        >>> hello()
        'Hey there!'

Binding generators

Python: Cython


  # hello.pyx
  def add(int i, int j):
      return i + j


  # setup.py
  from setuptools import setup
  from Cython.Build import cythonize

  setup(ext_modules=cythonize("hello.pyx"))


  $ python setup.py build_ext --inplace

C++: pybind11


  #include <pybind11/pybind11.h>

  int add(int i, int j) {
      return i + j;
  }

  PYBIND11_MODULE(example, m) {
    // optional module docstring
    m.doc() = "pybind11 example plugin";
    m.def("add", &add,
      "A function that adds two numbers");
  }


  $ c++ -O3 -Wall -shared -std=c++11 \
    -fPIC $(python3 -m pybind11 --includes) example.cpp \
    -o example$(python3-config --extension-suffix)

C++: shiboken


  // test.cpp
  #include <test.hpp>

  int add(int i, int j) {
      return i + j;
  }


  // test.hpp
  int add(int i, int j);


    <!-- bindings.xml -->
    <?xml version="1.0"?>
    <typesystem package="simple">
        <function signature="add(int, int)"/>
    </typesystem>


  $ cmake -S . -B build
  $ cmake --build build
  $ cmake --install build

Zig: Ziggy Pydust


  const py = @import("pydust");

  pub fn add(args: struct { a: i32, b: i32 }) i32 {
      return args.a + args.b;
  }

  comptime {
    py.rootmodule(@This());
  }


  $ poetry install
  $ poetry run python -c "from hello_zig import hello; print(hello())"

Rust: PyO3


use pyo3::prelude::*;

/// Formats the sum of two numbers as string.
#[pyfunction]
fn hello(s: &str) -> PyResult {
    let msg = format!("Hey there {s}!");
    Ok(msg)
}

/// A Python module implemented in Rust.
#[pymodule]
fn mymodule(m: &Bound<'_, PyModule>) -> PyResult<()> {
    m.add_function(wrap_pyfunction!(hello, m)?)?;
    Ok(())
}


  $ maturin develop

End of Second Part Presentation

Extending cpython

Exercise 06

Create a minimal `"Hello, World!"` extension in C

Exercise 07

Create a minimal `"Hello, World!"` extension in Zig

Exercise 08

Create a minimal `"Hello, World!"` extension in Rust

Exercise 09

Create a new module with one or more functions

Exercise 10

Re-write a Python functionality in Rust

Exercise 10 - Rewrite a Python functionality in Rust

Pick any functionality you like!
Try to keep it minimal and simple first
For example: glob.glob

Tutorial

Understanding and expanding Python: hands-on experience with the Python internals

Dr. Cristián Maureira-Fredes

@cmaureir

Understanding and expanding Python: hands-on experience with the Python internals

The Structure

of this tutorial

First Part Presentation

Navigating cpython

CPython is the standard implementation

To get around it

We need to understand C

a little bit

Python and C (1/3)

Python and C (2/3)

Python and C (3/3)

*

int *a;

C pointers (1/2)

C pointers (1/2)

The cpython repo structure

PyObject (1/2)

PyObject (2/2)

Python C-API

Strings

Lists

Check what's the PyObject storing

Parse arguments

How does CPython works?

From Python to Machine Code

End of First Part Presentation

Navigating cpython

Compiler Setup

Environment Setup

Exercise 00

Compile a "Hello, World!" example in C

Exercise 01

Build cpython

Exercise 01 - Build cpython

For Linux/macOS

For Windows

Exercise 02

Add a Python function to cpython

Exercise 02 - New Python function in cpython

Exercise 03

Add a new built-in function to cpython

Exercise 03 - New C function in cpython

Exercise 04

Add a new method to a Python container

Exercise 05

Add an alias to a Python keyword

Second Part Presentation

Extending cpython

Creating a Python C-extension

Parts of a C extension (1/4)

Parts of a C extension (2/4)

Parts of a C extension (3/4)

Parts of a C extension (4/4)

Binding generators

Python: Cython

C++: pybind11

C++: shiboken

Zig: Ziggy Pydust

Rust: PyO3

End of Second Part Presentation

Extending cpython

Exercise 06

Create a minimal "Hello, World!" extension in C

Exercise 07

Create a minimal "Hello, World!" extension in Zig

Exercise 08

Create a minimal "Hello, World!" extension in Rust

Exercise 09

Create a new module with one or more functions

Exercise 10

Re-write a Python functionality in Rust

Exercise 10 - Rewrite a Python functionality in Rust

Understanding and expanding Python: hands-on experience with the Python internals

`int *a;`

The `cpython` repo structure

Compile a `"Hello, World!"` example in C

Build `cpython`

Add a Python function to `cpython`

Add a new built-in function to `cpython`

Create a minimal `"Hello, World!"` extension in C

Create a minimal `"Hello, World!"` extension in Zig

Create a minimal `"Hello, World!"` extension in Rust