This file is indexed.

/usr/include/rdkit/RDBoost/python_streambuf.h is in librdkit-dev 201603.5-2.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
//
//  This file is part of the CCTBX distribution:
//     http://cctbx.sourceforge.net/
//  Downloaded from here:
//    http://cctbx.svn.sourceforge.net/viewvc/cctbx/trunk/boost_adaptbx/python_streambuf.h?revision=13619
//
//  Copyright (c) 2006, The Regents of the University of
//  California, through Lawrence Berkeley National Laboratory (subject to
//  receipt of any required approvals from the U.S. Dept. of Energy).  All
//  rights reserved.
//
//  The license is here:
//    http://cctbx.svn.sourceforge.net/viewvc/cctbx/trunk/boost_adaptbx/LICENSE_2_0.txt?revision=5148
//
#ifndef BOOST_ADAPTBX_PYTHON_STREAMBUF_H
#define BOOST_ADAPTBX_PYTHON_STREAMBUF_H
#include <RDGeneral/BoostStartInclude.h>
#include <boost/python/object.hpp>
#include <boost/python/str.hpp>
#include <boost/python/extract.hpp>

#include <boost/optional.hpp>
#include <boost/utility/typed_in_place_factory.hpp>
#include <RDGeneral/BoostEndInclude.h>

//#include <tbxx/error_utils.hpp>
#include <RDGeneral/Invariant.h>

#include <streambuf>
#include <iostream>

namespace boost_adaptbx {
namespace python {

namespace bp = boost::python;

/// A stream buffer getting data from and putting data into a Python file object
/** The aims are as follow:

    - Given a C++ function acting on a standard stream, e.g.

      \code
      void read_inputs(std::istream& input) {
        ...
        input >> something >> something_else;
      }
      \endcode

      and given a piece of Python code which creates a file-like object,
      to be able to pass this file object to that C++ function, e.g.

      \code
      import gzip
      gzip_file_obj = gzip.GzipFile(...)
      read_inputs(gzip_file_obj)
      \endcode

      and have the standard stream pull data from and put data into the Python
      file object.

    - When Python \c read_inputs() returns, the Python object is able to
      continue reading or writing where the C++ code left off.

    - Operations in C++ on mere files should be competitively fast compared
      to the direct use of \c std::fstream.


    \b Motivation

      - the standard Python library offer of file-like objects (files,
        compressed files and archives, network, ...) is far superior to the
        offer of streams in the C++ standard library and Boost C++ libraries.

      - i/o code involves a fair amount of text processing which is more
        efficiently prototyped in Python but then one may need to rewrite
        a time-critical part in C++, in as seamless a manner as possible.

    \b Usage

    This is 2-step:

      - a trivial wrapper function

        \code
          using boost_adaptbx::python::streambuf;
          void read_inputs_wrapper(streambuf& input)
          {
            streambuf::istream is(input);
            read_inputs(is);
          }

          def("read_inputs", read_inputs_wrapper);
        \endcode

        which has to be written every time one wants a Python binding for
        such a C++ function.

      - the Python side

        \code
          from boost.python import streambuf
          read_inputs(streambuf(python_file_obj=obj, buffer_size=1024))
        \endcode

        \c buffer_size is optional. See also: \c default_buffer_size

  Note: references are to the C++ standard (the numbers between parentheses
  at the end of references are margin markers).
*/
class streambuf : public std::basic_streambuf<char> {
 private:
  typedef std::basic_streambuf<char> base_t;

 public:
  /* The syntax
      using base_t::char_type;
     would be nicer but Visual Studio C++ 8 chokes on it
  */
  typedef base_t::char_type char_type;
  typedef base_t::int_type int_type;
  typedef base_t::pos_type pos_type;
  typedef base_t::off_type off_type;
  typedef base_t::traits_type traits_type;

  // work around Visual C++ 7.1 problem
  inline static int traits_type_eof() { return traits_type::eof(); }

  /// The default size of the read and write buffer.
  /** They are respectively used to buffer data read from and data written to
      the Python file object. It can be modified from Python.
  */
  const static std::size_t default_buffer_size = 1024;

  /// Construct from a Python file object
  /** if buffer_size is 0 the current default_buffer_size is used.
  */
  streambuf(bp::object& python_file_obj, std::size_t buffer_size_ = 0)
      : py_read(getattr(python_file_obj, "read", bp::object())),
        py_write(getattr(python_file_obj, "write", bp::object())),
        py_seek(getattr(python_file_obj, "seek", bp::object())),
        py_tell(getattr(python_file_obj, "tell", bp::object())),
        buffer_size(buffer_size_ != 0 ? buffer_size_ : default_buffer_size),
        write_buffer(0),
        pos_of_read_buffer_end_in_py_file(0),
        pos_of_write_buffer_end_in_py_file(buffer_size),
        farthest_pptr(0) {
    TEST_ASSERT(buffer_size != 0);
    /* Some Python file objects (e.g. sys.stdout and sys.stdin)
       have non-functional seek and tell. If so, assign None to
       py_tell and py_seek.
     */
    if (py_tell != bp::object()) {
      try {
        off_type py_pos = bp::extract<off_type>(py_tell());
        if (py_seek != bp::object()) {
          /* Make sure we can actually seek.
             bzip2 readers from python have a seek method, but it fails
             when they are in write mode.
           */
          py_seek(py_pos);
        }
      } catch (bp::error_already_set&) {
        py_tell = bp::object();
        py_seek = bp::object();
        /* Boost.Python does not do any Python exception handling whatsoever
           So we need to catch it by hand like so.
         */
        PyErr_Clear();
      }
    }

    if (py_write != bp::object()) {
      // C-like string to make debugging easier
      write_buffer = new char[buffer_size + 1];
      write_buffer[buffer_size] = '\0';
      setp(write_buffer, write_buffer + buffer_size);  // 27.5.2.4.5 (5)
      farthest_pptr = pptr();
    } else {
      // The first attempt at output will result in a call to overflow
      setp(0, 0);
    }

    if (py_tell != bp::object()) {
      off_type py_pos = bp::extract<off_type>(py_tell());
      pos_of_read_buffer_end_in_py_file = py_pos;
      pos_of_write_buffer_end_in_py_file = py_pos;
    }
  }

  /// Mundane destructor freeing the allocated resources
  virtual ~streambuf() {
    if (write_buffer) delete[] write_buffer;
  }

  /// C.f. C++ standard section 27.5.2.4.3
  /** It is essential to override this virtual function for the stream
      member function readsome to work correctly (c.f. 27.6.1.3, alinea 30)
   */
  virtual std::streamsize showmanyc() {
    int_type const failure = traits_type::eof();
    int_type status = underflow();
    if (status == failure) return -1;
    return egptr() - gptr();
  }

  /// C.f. C++ standard section 27.5.2.4.3
  virtual int_type underflow() {
    int_type const failure = traits_type::eof();
    if (py_read == bp::object()) {
      throw std::invalid_argument(
          "That Python file object has no 'read' attribute");
    }
    read_buffer = py_read(buffer_size);
    char* read_buffer_data;
    bp::ssize_t py_n_read;
    if (PyBytes_AsStringAndSize(read_buffer.ptr(), &read_buffer_data,
                                &py_n_read) == -1) {
      setg(0, 0, 0);
      throw std::invalid_argument(
          "The method 'read' of the Python file object "
          "did not return a string.");
    }
    off_type n_read = (off_type)py_n_read;
    pos_of_read_buffer_end_in_py_file += n_read;
    setg(read_buffer_data, read_buffer_data, read_buffer_data + n_read);
    // ^^^27.5.2.3.1 (4)
    if (n_read == 0) return failure;
    return traits_type::to_int_type(read_buffer_data[0]);
  }

  /// C.f. C++ standard section 27.5.2.4.5
  virtual int_type overflow(int_type c = traits_type_eof()) {
    if (py_write == bp::object()) {
      throw std::invalid_argument(
          "That Python file object has no 'write' attribute");
    }
    farthest_pptr = std::max(farthest_pptr, pptr());
    off_type n_written = (off_type)(farthest_pptr - pbase());
    bp::str chunk(pbase(), farthest_pptr);
    py_write(chunk);
    if (!traits_type::eq_int_type(c, traits_type::eof())) {
      py_write(traits_type::to_char_type(c));
      n_written++;
    }
    if (n_written) {
      pos_of_write_buffer_end_in_py_file += n_written;
      setp(pbase(), epptr());
      // ^^^ 27.5.2.4.5 (5)
      farthest_pptr = pptr();
    }
    return traits_type::eq_int_type(c, traits_type::eof())
               ? traits_type::not_eof(c)
               : c;
  }

  /// Update the python file to reflect the state of this stream buffer
  /** Empty the write buffer into the Python file object and set the seek
      position of the latter accordingly (C++ standard section 27.5.2.4.2).
      If there is no write buffer or it is empty, but there is a non-empty
      read buffer, set the Python file object seek position to the
      seek position in that read buffer.
  */
  virtual int sync() {
    int result = 0;
    farthest_pptr = std::max(farthest_pptr, pptr());
    if (farthest_pptr && farthest_pptr > pbase()) {
      off_type delta = pptr() - farthest_pptr;
      int_type status = overflow();
      if (traits_type::eq_int_type(status, traits_type::eof())) result = -1;
      if (py_seek != bp::object()) py_seek(delta, 1);
    } else if (gptr() && gptr() < egptr()) {
      if (py_seek != bp::object()) py_seek(gptr() - egptr(), 1);
    }
    return result;
  }

  /// C.f. C++ standard section 27.5.2.4.2
  /** This implementation is optimised to look whether the position is within
      the buffers, so as to avoid calling Python seek or tell. It is
      important for many applications that the overhead of calling into Python
      is avoided as much as possible (e.g. parsers which may do a lot of
      backtracking)
  */
  virtual pos_type seekoff(off_type off, std::ios_base::seekdir way,
                           std::ios_base::openmode which = std::ios_base::in |
                                                           std::ios_base::out) {
    /* In practice, "which" is either std::ios_base::in or out
       since we end up here because either seekp or seekg was called
       on the stream using this buffer. That simplifies the code
       in a few places.
    */
    int const failure = off_type(-1);

    if (py_seek == bp::object()) {
      throw std::invalid_argument(
          "That Python file object has no 'seek' attribute");
    }

    // we need the read buffer to contain something!
    if (which == std::ios_base::in && !gptr()) {
      if (traits_type::eq_int_type(underflow(), traits_type::eof())) {
        return failure;
      }
    }

    // compute the whence parameter for Python seek
    int whence;
    switch (way) {
      case std::ios_base::beg:
        whence = 0;
        break;
      case std::ios_base::cur:
        whence = 1;
        break;
      case std::ios_base::end:
        whence = 2;
        break;
      default:
        return failure;
    }

    // Let's have a go
    boost::optional<off_type> result =
        seekoff_without_calling_python(off, way, which);
    if (!result) {
      // we need to call Python
      if (which == std::ios_base::out) overflow();
      if (way == std::ios_base::cur) {
        if (which == std::ios_base::in)
          off -= egptr() - gptr();
        else if (which == std::ios_base::out)
          off += pptr() - pbase();
      }
      py_seek(off, whence);
      result = off_type(bp::extract<off_type>(py_tell()));
      if (which == std::ios_base::in) underflow();
    }
    return *result;
  }

  /// C.f. C++ standard section 27.5.2.4.2
  virtual pos_type seekpos(pos_type sp,
                           std::ios_base::openmode which = std::ios_base::in |
                                                           std::ios_base::out) {
    return streambuf::seekoff(sp, std::ios_base::beg, which);
  }

 private:
  bp::object py_read, py_write, py_seek, py_tell;

  std::size_t buffer_size;

  /* This is actually a Python string and the actual read buffer is
     its internal data, i.e. an array of characters. We use a Boost.Python
     object so as to hold on it: as a result, the actual buffer can't
     go away.
  */
  bp::object read_buffer;

  /* A mere array of char's allocated on the heap at construction time and
     de-allocated only at destruction time.
  */
  char* write_buffer;

  off_type pos_of_read_buffer_end_in_py_file,
      pos_of_write_buffer_end_in_py_file;

  // the farthest place the buffer has been written into
  char* farthest_pptr;

  boost::optional<off_type> seekoff_without_calling_python(
      off_type off, std::ios_base::seekdir way, std::ios_base::openmode which) {
    boost::optional<off_type> const failure;

    // Buffer range and current position
    off_type buf_begin, buf_end, buf_cur, upper_bound;
    off_type pos_of_buffer_end_in_py_file;
    if (which == std::ios_base::in) {
      pos_of_buffer_end_in_py_file = pos_of_read_buffer_end_in_py_file;
      buf_begin = reinterpret_cast<std::streamsize>(eback());
      buf_cur = reinterpret_cast<std::streamsize>(gptr());
      buf_end = reinterpret_cast<std::streamsize>(egptr());
      upper_bound = buf_end;
    } else if (which == std::ios_base::out) {
      pos_of_buffer_end_in_py_file = pos_of_write_buffer_end_in_py_file;
      buf_begin = reinterpret_cast<std::streamsize>(pbase());
      buf_cur = reinterpret_cast<std::streamsize>(pptr());
      buf_end = reinterpret_cast<std::streamsize>(epptr());
      farthest_pptr = std::max(farthest_pptr, pptr());
      upper_bound = reinterpret_cast<std::streamsize>(farthest_pptr) + 1;
    } else {
      CHECK_INVARIANT(0, "unreachable code");
    }

    // Sought position in "buffer coordinate"
    off_type buf_sought;
    if (way == std::ios_base::cur) {
      buf_sought = buf_cur + off;
    } else if (way == std::ios_base::beg) {
      buf_sought = buf_end + (off - pos_of_buffer_end_in_py_file);
    } else if (way == std::ios_base::end) {
      return failure;
    } else {
      CHECK_INVARIANT(0, "unreachable code");
    }

    // if the sought position is not in the buffer, give up
    if (buf_sought < buf_begin || buf_sought >= upper_bound) return failure;

    // we are in wonderland
    if (which == std::ios_base::in)
      gbump(buf_sought - buf_cur);
    else if (which == std::ios_base::out)
      pbump(buf_sought - buf_cur);
    return pos_of_buffer_end_in_py_file + (buf_sought - buf_end);
  }

 public:
  class istream : public std::istream {
   public:
    istream(streambuf& buf) : std::istream(&buf) {
      exceptions(std::ios_base::badbit);
    }

    ~istream() {
      // do nothing.
      // This used to do:
      // if (this->good()) this->sync();
      // but that caused problems if the underlying file had been closed
      // (see github #579) and really doesn't seem necessary for what we're
      // doing.
    }
  };

  class ostream : public std::ostream {
   public:
    ostream(streambuf& buf) : std::ostream(&buf) {
      exceptions(std::ios_base::badbit);
    }

    ~ostream() {
      if (this->good()) this->flush();
    }
  };
};

// std::size_t streambuf::default_buffer_size = 1024;

struct streambuf_capsule {
  streambuf python_streambuf;

  streambuf_capsule(bp::object& python_file_obj, std::size_t buffer_size = 0)
      : python_streambuf(python_file_obj, buffer_size) {}
};

struct ostream : private streambuf_capsule, streambuf::ostream {
  ostream(bp::object& python_file_obj, std::size_t buffer_size = 0)
      : streambuf_capsule(python_file_obj, buffer_size),
        streambuf::ostream(python_streambuf) {}

  ~ostream() {
    try {
      if (this->good()) this->flush();
    } catch (bp::error_already_set&) {
      PyErr_Clear();
      throw std::runtime_error(
          "Problem closing python ostream.\n"
          "  Known limitation: the error is unrecoverable. Sorry.\n"
          "  Suggestion for programmer: add ostream.flush() before"
          " returning.");
    }
  }
};
}
}  // boost_adaptbx::python

#endif  // GUARD