xpra icon
Bug tracker and wiki

This bug tracker and wiki are being discontinued
please use https://github.com/Xpra-org/xpra instead.


Ticket #110: xpra-videodecode-thread.patch

File xpra-videodecode-thread.patch, 23.1 KB (added by Antoine Martin, 9 years ago)

this moves picture decoding to a separate thread

  • xpra/x264/codec.pyx

     
    6666    def get_type(self):
    6767        return  "x264"
    6868
     69cdef class RGBImage:
     70    cdef uint8_t *data
     71    cdef int size
     72    cdef int rowstride
    6973
     74    cdef init(self, uint8_t *data, int size, int rowstride):
     75        self.data = data
     76        self.size = size
     77        self.rowstride = rowstride
     78
     79    def free(self):
     80        assert self.data!=NULL
     81        xmemfree(self.data)
     82        self.data = NULL
     83
     84    def get_data(self):
     85        return (<char *>self.data)[:self.size]
     86
     87    def get_size(self):
     88        return self.size
     89
     90    def get_rowstride(self):
     91        return self.rowstride
     92
     93
    7094cdef class Decoder(xcoder):
    7195    cdef uint8_t *last_image
    7296
    … …  
    123147        PyObject_AsReadBuffer(input, <const_void_pp> &buf, &buf_len)
    124148        padded_buf = <unsigned char *> xmemalign(buf_len+32)
    125149        if padded_buf==NULL:
    126             return 100, 0, ""
     150            return 100, None
    127151        memcpy(padded_buf, buf, buf_len)
    128152        memset(padded_buf+buf_len, 0, 32)
    129153        set_decoder_csc_format(self.context, int(options.get("csc_pixel_format", -1)))
    … …  
    133157                i = csc_image_yuv2rgb(self.context, yuvplanes, yuvstrides, &dout, &outsize, &outstride)
    134158        xmemfree(padded_buf)
    135159        if i!=0:
    136             return i, 0, ""
    137         self.last_image = dout
    138         doutv = (<char *>dout)[:outsize]
    139         return  i, outstride, doutv
     160            return i, None
     161        rgb_image = RGBImage()
     162        rgb_image.init(dout, outsize, outstride)
     163        return  i, rgb_image
    140164
    141     def free_image(self):
    142         assert self.last_image!=NULL
    143         xmemfree(self.last_image)
    144         self.last_image = NULL
    145165
    146 
    147166cdef class Encoder(xcoder):
    148167
    149168    cdef int supports_options

  • xpra/client_window.py

     
    380380        #the "ClientWindow"
    381381        self._client.send_refresh_all()
    382382
    383     def draw_region(self, x, y, width, height, coding, img_data, rowstride, packet_sequence, options):
    384         success = self._backing.draw_region(x, y, width, height, coding, img_data, rowstride, options)
    385         if success:
    386             queue_draw(self, x, y, width, height)
    387         #clear the auto refresh if enough pixels were sent (arbitrary limit..)
    388         if success and self._refresh_timer and width*height>16*16:
    389             gobject.source_remove(self._refresh_timer)
    390             self._refresh_timer = None
    391         #if we need to set a refresh timer, do it:
    392         is_hq = options.get("quality", 0)>=95
    393         is_lossy = coding in ("jpeg", "vpx", "x264")
    394         if self._refresh_timer is None and self._client.auto_refresh_delay>0 and is_lossy and not is_hq:
    395             #make sure our own refresh does not make us fire again
    396             #FIXME: this should be per-window!
    397             if self._refresh_ignore_sequence<packet_sequence:
    398                 self._refresh_ignore_sequence = packet_sequence+1
    399                 self._refresh_timer = gobject.timeout_add(int(1000 * self._client.auto_refresh_delay), self.refresh_window)
    400         return success
     383    def draw_region(self, x, y, width, height, coding, img_data, rowstride, packet_sequence, options, callbacks):
     384        def after_draw_refresh(success):
     385            log("after_draw_refresh(%s)", success)
     386            if success:
     387                queue_draw(self, x, y, width, height)
     388            #clear the auto refresh if enough pixels were sent (arbitrary limit..)
     389            if success and self._refresh_timer and width*height>16*16:
     390                gobject.source_remove(self._refresh_timer)
     391                self._refresh_timer = None
     392            #if we need to set a refresh timer, do it:
     393            is_hq = options.get("quality", 0)>=95
     394            is_lossy = coding in ("jpeg", "vpx", "x264")
     395            if self._refresh_timer is None and self._client.auto_refresh_delay>0 and is_lossy and not is_hq:
     396                #make sure our own refresh does not make us fire again
     397                #FIXME: this should be per-window!
     398                if self._refresh_ignore_sequence<packet_sequence:
     399                    self._refresh_ignore_sequence = packet_sequence+1
     400                    self._refresh_timer = gobject.timeout_add(int(1000 * self._client.auto_refresh_delay), self.refresh_window)
     401        callbacks.append(after_draw_refresh)
     402        self._backing.draw_region(x, y, width, height, coding, img_data, rowstride, options, callbacks)
    401403
    402404    """ gtk3 """
    403405    def do_draw(self, context):

  • xpra/vpx/codec.pyx

     
    5151    def get_height(self):
    5252        return self.height
    5353
     54cdef class RGBImage:
     55    cdef uint8_t *data
     56    cdef int size
     57    cdef int rowstride
    5458
     59    cdef init(self, uint8_t *data, int size, int rowstride):
     60        self.data = data
     61        self.size = size
     62        self.rowstride = rowstride
     63
     64    def free(self):
     65        assert self.data!=NULL
     66        xmemfree(self.data)
     67        self.data = NULL
     68
     69    def get_data(self):
     70        return (<char *>self.data)[:self.size]
     71
     72    def get_size(self):
     73        return self.size
     74
     75    def get_rowstride(self):
     76        return self.rowstride
     77
     78
    5579cdef class Decoder(xcoder):
    5680    cdef uint8_t *last_image
    5781
    … …  
    97121        PyObject_AsReadBuffer(input, <const_void_pp> &buf, &buf_len)
    98122        i = decompress_image(self.context, buf, buf_len, &yuvplanes, &outsize, &yuvstrides)
    99123        if i!=0:
    100             return i, 0, ""
     124            return i, None
    101125        with nogil:
    102126            i = csc_image_yuv2rgb(self.context, yuvplanes, yuvstrides, &dout, &outsize, &outstride)
    103127        if i!=0:
    104             return i, 0, ""
    105         self.last_image = dout
    106         doutv = (<char *>dout)[:outsize]
    107         return  i, outstride, doutv
     128            return i, None
     129        rgb_image = RGBImage()
     130        rgb_image.init(dout, outsize, outstride)
     131        return  i, rgb_image
    108132
    109     def free_image(self):
    110         assert self.last_image!=NULL
    111         xmemfree(self.last_image)
    112         self.last_image = NULL
    113133
    114 
    115134cdef class Encoder(xcoder):
    116135
    117136    def init_context(self, width, height, supports_options):    #@DuplicatedSignature

  • xpra/client.py

     
    4141import os
    4242import time
    4343import ctypes
     44from threading import Thread
     45try:
     46    from queue import Queue     #@UnresolvedImport @UnusedImport (python3)
     47except:
     48    from Queue import Queue     #@Reimport
    4449
     50
    4551from wimpiggy.util import (n_arg_signal,
    4652                           gtk_main_quit_really,
    4753                           gtk_main_quit_on_fatal_exceptions_enable)
    … …  
    98104            self.jpegquality = 50
    99105        self.dpi = int(opts.dpi)
    100106
     107        #draw thread:
     108        self._draw_queue = Queue(maxsize=1)
     109        self._draw_thread = Thread(target=self._draw_thread_loop, name="draw_loop")
     110        self._draw_thread.setDaemon(True)
     111        self._draw_thread.start()
     112
    101113        #statistics:
    102114        self.server_start_time = -1
    103115        self.server_platform = ""
    … …  
    767779            window.resize(w, h)
    768780
    769781    def _process_draw(self, packet):
    770         (wid, x, y, width, height, coding, data, packet_sequence, rowstride) = packet[1:10]
    771         options = {}
    772         if len(packet)>10:
    773             options = packet[10]
    774         log("process_draw %s bytes for window %s using %s encoding with options=%s", len(data), wid, coding, options)
     782        wid = packet[1]
    775783        window = self._id_to_window.get(wid)
    776         decode_time = 0
    777         if window:
    778             start = time.time()
    779             if window.draw_region(x, y, width, height, coding, data, rowstride, packet_sequence, options):
    780                 end = time.time()
    781                 self.pixel_counter.append((end, width*height))
    782                 decode_time = int(end*1000*1000-start*1000*1000)
    783         else:
     784        if not window:
    784785            #window is gone
     786            width, height, coding, data, packet_sequence = packet[4:9]
    785787            if coding=="mmap":
    786788                #we need to ack the data to free the space!
    787789                assert self.mmap_enabled
    788790                data_start = ctypes.c_uint.from_buffer(self.mmap, 0)
     791                data = packet[7]
    789792                offset, length = data[-1]
    790793                data_start.value = offset+length
     794            self.send_damage_sequence(wid, packet_sequence, width, height, -1)
     795            return
     796        self._draw_queue.put(packet)
     797
     798    def send_damage_sequence(self, wid, packet_sequence, width, height, decode_time):
    791799        self.send_now(["damage-sequence", packet_sequence, wid, width, height, decode_time])
    792800
     801    def _draw_thread_loop(self):
     802        while self.exit_code is None:
     803            packet = self._draw_queue.get()
     804            try:
     805                self._do_draw(packet)
     806            except:
     807                log.error("error processing draw packet", exc_info=True)
     808
     809    def _do_draw(self, packet):
     810        """ this runs from the draw thread above """
     811        wid, x, y, width, height, coding, data, packet_sequence, rowstride = packet[1:10]
     812        window = self._id_to_window.get(wid)
     813        if not window:
     814            return
     815        options = {}
     816        if len(packet)>10:
     817            options = packet[10]
     818        log("process_draw %s bytes for window %s using %s encoding with options=%s", len(data), wid, coding, options)
     819        start = time.time()
     820        def record_decode_time(success):
     821            if success:
     822                end = time.time()
     823                decode_time = int(end*1000*1000-start*1000*1000)
     824                self.pixel_counter.append((end, width*height))
     825            else:
     826                decode_time = -1
     827            log("record_decode_time(%s) wid=%s, %s: %sx%s, %sms", success, wid, coding, width, height, int(decode_time/100)/10.0)
     828            self.send_damage_sequence(wid, packet_sequence, width, height, decode_time)
     829        window.draw_region(x, y, width, height, coding, data, rowstride, packet_sequence, options, [record_decode_time])
     830
    793831    def _process_cursor(self, packet):
    794832        if not self.cursors_enabled:
    795833            return

  • xpra/window_backing.py

     
    99
    1010import ctypes
    1111import cairo
     12import gobject
    1213
    1314from wimpiggy.log import Logger
    1415log = Logger()
    … …  
    5354            assert len(img_data) == width * 3 * height
    5455        return Image.fromstring("RGB", (width, height), img_data, 'raw', 'RGB', rowstride, 1)
    5556
    56     def paint_rgb24(self, img_data, x, y, width, height, rowstride):
     57    def fire_paint_callbacks(self, callbacks, success):
     58        for x in callbacks:
     59            try:
     60                x(success)
     61            except:
     62                log.error("error calling %s(%s)", x, success, exc_info=True)
     63
     64    def paint_rgb24(self, img_data, x, y, width, height, rowstride, options, callbacks):
    5765        raise Exception("override me!")
    58     def paint_png(self, img_data, x, y, width, height):
    59         raise Exception("override me!")
    6066
    61     def paint_x264(self, img_data, x, y, width, height, rowstride, options):
     67    def paint_x264(self, img_data, x, y, width, height, rowstride, options, callbacks):
    6268        assert "x264" in ENCODINGS
    6369        from xpra.x264.codec import Decoder     #@UnresolvedImport
    64         return  self.paint_with_video_decoder(Decoder, "x264", img_data, x, y, width, height, rowstride, options)
     70        self.paint_with_video_decoder(Decoder, "x264", img_data, x, y, width, height, rowstride, options, callbacks)
    6571
    66     def paint_vpx(self, img_data, x, y, width, height, rowstride, options):
     72    def paint_vpx(self, img_data, x, y, width, height, rowstride, options, callbacks):
    6773        assert "vpx" in ENCODINGS
    6874        from xpra.vpx.codec import Decoder     #@UnresolvedImport
    69         return  self.paint_with_video_decoder(Decoder, "vpx", img_data, x, y, width, height, rowstride, options)
     75        self.paint_with_video_decoder(Decoder, "vpx", img_data, x, y, width, height, rowstride, options, callbacks)
    7076
    71     def paint_with_video_decoder(self, factory, coding, img_data, x, y, width, height, rowstride, options):
     77    def paint_with_video_decoder(self, factory, coding, img_data, x, y, width, height, rowstride, options, callbacks):
    7278        assert x==0 and y==0
    7379        if self._video_decoder:
    7480            if self._video_decoder.get_type()!=coding:
    … …  
    8288            self._video_decoder = factory()
    8389            self._video_decoder.init_context(width, height, options)
    8490        log("paint_with_video_decoder: options=%s", options)
    85         err, outstride, data = self._video_decoder.decompress_image_to_rgb(img_data, options)
    86         if err!=0:
     91        err, rgb_image = self._video_decoder.decompress_image_to_rgb(img_data, options)
     92        if err!=0 or rgb_image is None or rgb_image.get_size()==0:
    8793            log.error("paint_with_video_decoder: ouch, decompression error %s", err)
    8894            return  False
    89         if not data:
    90             log.error("paint_with_video_decoder: ouch, no data from %s decoder", coding)
     95        def paint_then_free():
     96            try:
     97                log("paint_with_video_decoder: decompressed %s to %s bytes (%s%%) of rgb24 (%s*%s*3=%s) (outstride: %s)", len(img_data), rgb_image.get_size(), int(100*len(img_data)/rgb_image.get_size()),width, height, width*height*3, rgb_image.get_rowstride())
     98                self.paint_rgb24(rgb_image.get_data(), x, y, width, height, rgb_image.get_rowstride(), options, callbacks)
     99            finally:
     100                rgb_image.free()
    91101            return  False
    92         try:
    93             log("paint_with_video_decoder: decompressed %s to %s bytes (%s%%) of rgb24 (%s*%s*3=%s) (outstride: %s)", len(img_data), len(data), int(100*len(img_data)/len(data)),width, height, width*height*3, outstride)
    94             self.paint_rgb24(data, x, y, width, height, outstride)
    95             return  True
    96         finally:
    97             self._video_decoder.free_image()
     102        gobject.idle_add(paint_then_free)
     103        return  False
    98104
    99105
    100106"""
    … …  
    136142        Backing.close(self)
    137143        self._backing.finish()
    138144
    139     def paint_png(self, img_data, x, y, width, height):
     145    def paint_png(self, img_data, x, y, width, height, rowstride, options, callbacks):
     146        """ must be called from UI thread """
    140147        try:
    141148            from io import BytesIO          #@Reimport
    142149            import sys
    … …  
    153160        gc.set_source_surface(surf)
    154161        gc.paint()
    155162        surf.finish()
    156         return  True
     163        self.fire_paint_callbacks(callbacks, True)
     164        return  False
    157165
    158     def paint_pil_image(self, pil_image, width, height):
     166    def paint_pil_image(self, pil_image, width, height, rowstride, options, callbacks):
    159167        try:
    160168            from io import BytesIO
    161169            buf = BytesIO()
    … …  
    165173        pil_image.save(buf, format="PNG")
    166174        png_data = buf.getvalue()
    167175        buf.close()
    168         self.cairo_paint_png(png_data, 0, 0, width, height)
    169         return  True
     176        gobject.idle_add(self.paint_png, png_data, 0, 0, width, height, rowstride, options, callbacks)
    170177
    171     def paint_rgb24(self, img_data, x, y, width, height, rowstride):
    172         log("cairo_paint_rgb24(..,%s,%s,%s,%s,%s)", x, y, width, height, rowstride)
     178    def paint_rgb24(self, img_data, x, y, width, height, rowstride, options, callbacks):
     179        """ must be called from UI thread """
     180        log("cairo_paint_rgb24(..,%s,%s,%s,%s,%s,%s,%s)", x, y, width, height, rowstride, options, callbacks)
    173181        gc = cairo.Context(self._backing)
    174182        if rowstride==0:
    175183            rowstride = width*3
    … …  
    177185        gc.set_source_surface(surf)
    178186        gc.paint()
    179187        surf.finish()
    180         return  True
     188        self.fire_paint_callbacks(callbacks, True)
     189        return  False
    181190
    182     def paint_mmap(self, img_data, x, y, width, height, rowstride):
     191    def paint_mmap(self, img_data, x, y, width, height, rowstride, options, callbacks):
    183192        """ see _mmap_send() in server.py for details """
    184193        assert "rgb24" in ENCODINGS
    185194        assert self.mmap_enabled
    … …  
    200209                data += self.mmap.read(length)
    201210                data_start.value = offset+length
    202211            image = self.rgb24image(data, width, height, rowstride)
    203         return  self.paint_pil_image(image, width, height)
     212        self.paint_pil_image(image, width, height, rowstride, options, callbacks)
     213        return  False
    204214
    205     def draw_region(self, x, y, width, height, coding, img_data, rowstride, options):
    206         log.debug("draw_region(%s,%s,%s,%s,%s,..,%s,%s)", x, y, width, height, coding, rowstride, options)
     215    def draw_region(self, *args):
     216        #FIXME: I am lazy and gtk3 support is lagging anyway:
     217        gobject.idle_add(self.do_draw_region, *args)
     218
     219    def do_draw_region(self, x, y, width, height, coding, img_data, rowstride, options, callbacks):
     220        log.debug("do_draw_region(%s,%s,%s,%s,%s,..,%s,%s,%s)", x, y, width, height, coding, rowstride, options, callbacks)
    207221        if coding == "mmap":
    208             return  self.paint_mmap(img_data, x, y, width, height, rowstride)
     222            return  self.paint_mmap(img_data, x, y, width, height, rowstride, options, callbacks)
    209223        elif coding in ["rgb24", "jpeg"]:
    210224            assert coding in ENCODINGS
    211225            if coding=="rgb24":
    212                 image = self.rgb24image(img_data, width, height, rowstride)
     226                image = self.rgb24image(img_data, width, height, rowstride, options, callbacks)
    213227            else:   #if coding=="jpeg":
    214                 image = self.jpegimage(img_data, width, height)
    215             return  self.paint_pil_image(image, width, height)
     228                image = self.jpegimage(img_data, width, height, rowstride, options, callbacks)
     229            return  self.paint_pil_image(image, width, height, rowstride, options, callbacks)
    216230        elif coding == "png":
    217231            assert coding in ENCODINGS
    218             return  self.paint_png(img_data, x, y, width, height)
     232            gobject.idle_add(self.paint_png, img_data, x, y, width, height, rowstride, options, callbacks)
    219233        raise Exception("invalid picture encoding: %s" % coding)
    220234
    221235    def cairo_draw(self, context, x, y):
    … …  
    257271        cr.set_source_rgb(1, 1, 1)
    258272        cr.fill()
    259273
    260     def paint_rgb24(self, img_data, x, y, width, height, rowstride):
     274    def paint_rgb24(self, img_data, x, y, width, height, rowstride, options, callbacks):
     275        """ must be called from UI thread """
    261276        assert "rgb24" in ENCODINGS
    262277        gc = self._backing.new_gc()
    263278        self._backing.draw_rgb_image(gc, x, y, width, height, gdk.RGB_DITHER_NONE, img_data, rowstride)
    264         return  True
     279        self.fire_paint_callbacks(callbacks, True)
     280        return  False
    265281
    266     def paint_pixbuf(self, coding, img_data, x, y, width, height, rowstride):
     282    def paint_pixbuf(self, coding, img_data, x, y, width, height, rowstride, options, callbacks):
     283        """ must be called from UI thread """
    267284        assert coding in ENCODINGS
    268285        loader = gdk.PixbufLoader(coding)
    269286        loader.write(img_data, len(img_data))
    … …  
    271288        pixbuf = loader.get_pixbuf()
    272289        if not pixbuf:
    273290            log.error("failed %s pixbuf=%s data len=%s" % (coding, pixbuf, len(img_data)))
    274             return  False
    275         gc = self._backing.new_gc()
    276         self._backing.draw_pixbuf(gc, pixbuf, 0, 0, x, y, width, height)
    277         return  True
     291            self.fire_paint_callbacks(callbacks, False)
     292        else:
     293            gc = self._backing.new_gc()
     294            self._backing.draw_pixbuf(gc, pixbuf, 0, 0, x, y, width, height)
     295            self.fire_paint_callbacks(callbacks, True)
     296        return  False
    278297
    279     def paint_mmap(self, img_data, x, y, width, height, rowstride):
     298    def paint_mmap(self, img_data, x, y, width, height, rowstride, options, callbacks):
     299        """ must be called from UI thread """
     300        #we could run just paint_rgb24 from the UI thread,
     301        #but this would not make much of a difference
     302        #and would complicate the code (add a callback to free mmap area)
    280303        """ see _mmap_send() in server.py for details """
    281304        assert self.mmap_enabled
    282305        data_start = ctypes.c_uint.from_buffer(self.mmap, 0)
    … …  
    285308            offset, length = img_data[0]
    286309            arraytype = ctypes.c_char * length
    287310            data = arraytype.from_buffer(self.mmap, offset)
    288             success = self.paint_rgb24(data, x, y, width, height, rowstride)
     311            self.paint_rgb24(data, x, y, width, height, rowstride, options, callbacks)
    289312            data_start.value = offset+length
    290313        else:
    291314            #re-construct the buffer from discontiguous chunks:
    … …  
    295318                self.mmap.seek(offset)
    296319                data += self.mmap.read(length)
    297320                data_start.value = offset+length
    298             success = self.paint_rgb24(data, x, y, width, height, rowstride)
    299         return  success
     321            self.paint_rgb24(data, x, y, width, height, rowstride, options, callbacks)
     322        return  False
    300323
    301     def draw_region(self, x, y, width, height, coding, img_data, rowstride, options):
    302         log("draw_region(%s, %s, %s, %s, %s, %s bytes, %s, %s)", x, y, width, height, coding, len(img_data), rowstride, options)
     324    def draw_region(self, x, y, width, height, coding, img_data, rowstride, options, callbacks):
     325        log("draw_region(%s, %s, %s, %s, %s, %s bytes, %s, %s, %s)", x, y, width, height, coding, len(img_data), rowstride, options, callbacks)
    303326        if coding == "mmap":
    304             return self.paint_mmap(img_data, x, y, width, height, rowstride)
     327            gobject.idle_add(self.paint_mmap, img_data, x, y, width, height, rowstride, options, callbacks)
    305328        elif coding == "rgb24":
    306329            if rowstride==0:
    307330                rowstride = width * 3
    308331            assert len(img_data) == rowstride * height, "expected %s bytes but received %s" % (rowstride * height, len(img_data))
    309             return self.paint_rgb24(img_data, x, y, width, height, rowstride)
     332            gobject.idle_add(self.paint_rgb24, img_data, x, y, width, height, rowstride, options, callbacks)
    310333        elif coding == "x264":
    311             return self.paint_x264(img_data, x, y, width, height, rowstride, options)
     334            self.paint_x264(img_data, x, y, width, height, rowstride, options, callbacks)
    312335        elif coding == "vpx":
    313             return self.paint_vpx(img_data, x, y, width, height, rowstride, options)
     336            self.paint_vpx(img_data, x, y, width, height, rowstride, options, callbacks)
    314337        else:
    315             return self.paint_pixbuf(coding, img_data, x, y, width, height, rowstride)
     338            gobject.idle_add(self.paint_pixbuf, coding, img_data, x, y, width, height, rowstride, options, callbacks)
    316339
    317340    def cairo_draw(self, context, x, y):
    318341        try: