Find for a given email an openpgp key in its DNS zone .

def lookup_identity(self, identity, type_):
        """Find for a given email an openpgp key in its DNS zone."""
        clean = clean_email_address(identity)
        local_part, domain = clean[0].split('@', 2)
        qname = compute_qname(local_part, domain)
        ns = resolver.ns_for(domain)
        if not ns:
            log.warn('No nameservers found for domain {}'.format(domain))
            return self.empty_result

        # XXX use a random one
        use_ns = ns[0]
        resolv = resolver.Resolver(timeout=self.resolve_timeout)
        # XXX use of dnssec validation
        query = resolv.query_at(qname, 'OPENPGPKEY', use_ns)
        response = query.get()
        if not response:
            log.warn('No response for OPENPGPKEY dns query')
            return self.empty_result
        rrsets = response.rrset
        if not rrsets:
            log.warn('No rrsets for OPENPGPKEY dns query')
            return self.empty_result
        # XXX many rrset, key and signature, must be considered
        key = self._extract_key(rrsets[0])
        keys = self._parse_key(key)
        return DiscoveryResult(keys)


Extract an armored representation of key from dns record .

def _extract_key(self, record):
        """Extract an armored representation of key from dns record."""
        return record.data

def _extract_key ( self , record ) : return record . data


def __init__(self):
        """Instanciate a `ContactDiscoveryResult`."""
        self.keys = []
        self.emails = []
        self.identities = []

def __init__ ( self ) : self . keys = [ ] self . emails = [ ] self . identities = [ ]


def __init__(self):
        """Instanciate a `ContactPublicKeyManager`."""
        conf = Configuration('global').configuration
        self.discoverer = PublicKeyDiscoverer(conf)

def __init__ ( self ) : conf = Configuration ( 'global' ) . configuration self . discoverer = PublicKeyDiscoverer ( conf )


Find keys related to a new identifier for a contact .

def _find_keys(self, user, contact, new_identifier, type_):
        """Find keys related to a new identifier for a contact."""
        results = self.discoverer.lookup_identity(new_identifier, type_)
        current_keys = PublicKey.find(user, contact.contact_id)
        known_keys = [x.fingerprint for x in current_keys]
        if results:
            for result in results:
                for key in result.keys:
                    if key.fingerprint not in known_keys:
                        log.info('Found new key {0}'.format(key.fingerprint))
                        yield result
                    else:
                        log.info('Known key {0}'.format(key.fingerprint))

def
_find_keys
(
self
,
user
,
contact
,
new_identifier
,
type_
)
:
results
=
self
.
discoverer
.
lookup_identity
(
new_identifier
,
type_
)
current_keys
=
PublicKey
.
find
(
user
,
contact
.
contact_id
)
known_keys
=
[
x
.
fingerprint
for
x
in
current_keys
]
if
results
:
for
result
in
results
:
for
key
in
result
.
keys
:
if
key
.
fingerprint
not
in
known_keys
:
log
.
info
(
'Found new key {0}'
.
format
(
key
.
fingerprint
)
)
yield
result
else
:
log
.
info
(
'Known key {0}'
.
format
(
key
.
fingerprint
)
)


Find new emails related to a contact public key .

def _filter_new_emails(self, contact, key):
        """Find new emails related to a contact public key."""
        known_emails = [x.address for x in contact.emails]
        for userid in key.userids:
            if userid.email not in known_emails:
                known_emails.append(userid.email)
                yield userid

def _filter_new_emails ( self , contact , key ) : known_emails = [ x . address for x in contact . emails ] for userid in key . userids : if userid . email not in known_emails : known_emails . append ( userid . email ) yield userid


Create new identities found for a contact .

def _filter_new_identities(self, contact, identities):
        """Create new identities found for a contact."""
        known_ids = [(x.type, x.name) for x in contact.identities]
        for identity in identities:
            if identity not in known_ids:
                yield identity

def _filter_new_identities ( self , contact , identities ) : known_ids = [ ( x . type , x . name ) for x in contact . identities ] for identity in identities : if identity not in known_ids : yield identity


def process_identity(self, user, contact, new_identifier, type_):
        """Process a contact new identity."""
        result = ContactDiscoveryResult()
        discovers = self._find_keys(user, contact, new_identifier, type_)
        found_emails = []
        found_ids = []
        for disco in discovers:
            for key in disco.keys:
                result.keys.append(key)
                found_emails.extend(self._filter_new_emails(contact, key))
            if disco.identities:
                ids = self._filter_new_identities(contact, disco.identities)
                found_ids.extend(ids)
        for email in found_emails:
            if email.email not in [x.email for x in result.emails]:
                result.emails.append(email)
        for identity in found_ids:
            if identity not in result.identities:
                result.identities.append(identity)
        return result

def process_identity ( self , user , contact , new_identifier , type_ ) : result = ContactDiscoveryResult ( ) discovers = self . _find_keys ( user , contact , new_identifier , type_ ) found_emails = [ ] found_ids = [ ] for disco in discovers : for key in disco . keys : result . keys . append ( key ) found_emails . extend ( self . _filter_new_emails ( contact , key ) ) if disco . identities : ids = self . _filter_new_identities ( contact , disco . identities ) found_ids . extend ( ids ) for email in found_emails : if email . email not in [ x . email for x in result . emails ] : result . emails . append ( email ) for identity in found_ids : if identity not in result . identities : result . identities . append ( identity ) return result


def lookup_identity(self, identifier, type_):
        """Find by name and type."""
        if type_ not in self._types:
            raise Exception('Invalid identity type {}'.format(type_))
        users = []
        find = self._fetch_identity(identifier, type_)
        if find:
            log.debug('Got keybase result : {}'.format(find))
            users.extend(find)
        keys = []
        identities = []
        for user in users:
            user_key = self._get_public_key(user['username'])
            if user_key:
                keys.append(user_key)
            for ident in self._types:
                if user['remote_proofs'].get(ident) and type_ != ident:
                    identities.append((ident, user['remote_proofs'][ident]))
        public_keys = []
        for key in keys:
            public_keys.extend(self._parse_key(key))
        return DiscoveryResult(public_keys, identities)

def
lookup_identity
(
self
,
identifier
,
type_
)
:
if
type_
not
in
self
.
_types
:
raise
Exception
(
'Invalid identity type {}'
.
format
(
type_
)
)
users
=
[
]
find
=
self
.
_fetch_identity
(
identifier
,
type_
)
if
find
:
log
.
debug
(
'Got keybase result : {}'
.
format
(
find
)
)
users
.
extend
(
find
)
keys
=
[
]
identities
=
[
]
for
user
in
users
:
user_key
=
self
.
_get_public_key
(
user
[
'username'
]
)
if
user_key
:
keys
.
append
(
user_key
)
for
ident
in
self
.
_types
:
if
user
[
'remote_proofs'
]
.
get
(
ident
)
and
type_
!=
ident
:
identities
.
append
(
(
ident
,
user
[
'remote_proofs'
]
[
ident
]
)
)
public_keys
=
[
]
for
key
in
keys
:
public_keys
.
extend
(
self
.
_parse_key
(
key
)
)
return
DiscoveryResult
(
public_keys
,
identities
)


Format a clean user name depending on type .

def _clean_name(self, name, type_):
        """Format a clean user name depending on type."""
        if type_ == 'twitter':
            if name.startswith('@'):
                return name.lstrip('@').lower()
        return name.lower()

def _clean_name ( self , name , type_ ) : if type_ == 'twitter' : if name . startswith ( '@' ) : return name . lstrip ( '@' ) . lower ( ) return name . lower ( )


Make a user discover API call on keybase for a name and type .

def _fetch_identity(self, name, type_):
        """Make a user discover API call on keybase for a name and type."""
        clean_name = self._clean_name(name, type_)
        url = '{}/user/discover.json?{}={}'. \
              format(self.url, type_, clean_name)
        log.debug('Will query keybase url {}'.format(url))
        res = requests.get(url, headers=self.headers, timeout=self.timeout)
        if res.status_code != 200:
            log.error('Keybase discover status {} for {} on {}'.
                      format(res.status_code, clean_name, type_))
            return []
        result = res.json()
        if not result.get('matches'):
            log.debug('No match for keybase discovery of {} {}'.
                      format(clean_name, type_))
            return []
        matches = result['matches'].get(type_, [[]])
        res = []
        for match in matches:
            res.extend(match)
        return res


def _get_public_key(self, username):
        """Fetch a public key for an user."""
        url = '{}/{}/key.asc'.format(self.base_url, username)
        log.debug('Will query keybase url {}'.format(url))
        res = requests.get(url, headers=self.headers, timeout=self.timeout)
        if res.status_code != 200:
            log.error('Keybase user key fetch status {} for {}'.
                      format(res.status_code, username))
            return None
        return res.text


Initialize a data provider connecting it to store .

def __init__(self, config):
        """Initialize a data provider, connecting it to store."""
        self.config = config
        self._store = self._connect_store()
        self._search = None
        self.iterator = None

def __init__ ( self , config ) : self . config = config self . _store = self . _connect_store ( ) self . _search = None self . iterator = None


def prepare(self, query, **kwargs):
        """Prepare the query to be iterated."""
        self._search = self._prepare(query, **kwargs)
        self.iterator = self._execute(**kwargs)

def prepare ( self , query , * * kwargs ) : self . _search = self . _prepare ( query , * * kwargs ) self . iterator = self . _execute ( * * kwargs )


def next(self):
        """Iterator method over search results."""
        if not self.iterator:
            raise StopIteration
        for item in self.iterator:
            yield self._format_item(item)

def next ( self ) : if not self . iterator : raise StopIteration for item in self . iterator : yield self . _format_item ( item )


def __init__(self, config):
        """Create a new FileDataProvider."""
        super(FileDataProvider, self).__init__(config)
        self._filename = None

def __init__ ( self , config ) : super ( FileDataProvider , self ) . __init__ ( config ) self . _filename = None


Format a result item from store into processing format .

def _format_item(self, item):
        """Format a result item from store into processing format."""
        raise NotImplementedError

def _format_item ( self , item ) : raise NotImplementedError


Iterator method to retrieve results from provider .

def next(self):
        """Iterator method to retrieve results from provider."""
        raise NotImplementedError

def next ( self ) : raise NotImplementedError


Save an input file as dest on object store .

def save_file(config, input_file, dest):
    """Save an input file as dest on object store."""
    conf = config['object_store']
    bucket = conf['buckets']['learn_models']
    client = Minio(conf["endpoint"],
                   access_key=conf["access_key"],
                   secret_key=conf["secret_key"],
                   secure=False)
    try:
        client.make_bucket(bucket, conf['location'])
    except BucketAlreadyOwnedByYou:
        pass
    except BucketAlreadyExists:
        pass
    except Exception as exc:
        raise exc
    try:
        resp = client.fput_object(bucket, dest, input_file)
        log.info('Put file {0}: {1}'.format(input_file, resp))
    except Exception as exc:
        log.info('Unable to save file in object store %r' % exc)
        raise exc


def __new__(cls, *args, **kwargs):
        """Create singleton instance of class."""
        if not isinstance(cls._instance, cls):
            cls._instance = object.__new__(cls, *args, **kwargs)
        return cls._instance

def __new__ ( cls , * args , * * kwargs ) : if not isinstance ( cls . _instance , cls ) : cls . _instance = object . __new__ ( cls , * args , * * kwargs ) return cls . _instance


def __init__(self, filename):
        """A singleton around geoip reader."""
        self.reader = geoip.Reader(filename)

def __init__ ( self , filename ) : self . reader = geoip . Reader ( filename )


Instanciate a new qualifier for an user .

def __init__(self, user, conf=None):
        """Instanciate a new qualifier for an user."""
        self.user = user
        self.conf = conf
        self._features = init_features('device')

def __init__ ( self , user , conf = None ) : self . user = user self . conf = conf self . _features = init_features ( 'device' )


Process a device to compute it s privacy features and PI .

def process(self, device):
        """Process a device to compute it's privacy features and PI."""
        features = {}
        if device.user_agent:
            ua = parse_ua(device.user_agent)
            browser_version = ua.browser.version_string.lower()
            features.update({'browser_family': ua.browser.family.lower(),
                             'browser_version': browser_version,
                             'os_family': ua.os.family.lower(),
                             'os_version': ua.os.version_string.lower(),
                             'device_family': ua.device.family.lower(),
                             'device_type': self._get_device_type(ua)})
        # XXX processing IP address to detect some informations
        if device.ip_creation:
            features.update(self._process_ip_address(device.ip_creation))
        else:
            log.warn('No ip address found for device')
        return None, features


def _get_device_type(self, ua):
        """Return guessed type of user agent."""
        if ua.is_mobile:
            return 'smartphone'
        if ua.is_tablet:
            return 'tablet'
        if ua.is_pc:
            return 'desktop'
        return 'other'

def _get_device_type ( self , ua ) : if ua . is_mobile : return 'smartphone' if ua . is_tablet : return 'tablet' if ua . is_pc : return 'desktop' return 'other'


Get a MailMessage instance and extract privacy features .

def __init__(self, message, config):
        """Get a ``MailMessage`` instance and extract privacy features."""
        self.message = message
        self.config = config
        self._features = init_features('message')

def __init__ ( self , message , config ) : self . message = message self . config = config self . _features = init_features ( 'message' )


def is_blacklist_mx(self, mx):
        """MX is blacklisted."""
        blacklisted = self.config.get('blacklistes.mx')
        if not blacklisted:
            return False
        if mx in blacklisted:
            return True
        return False

def is_blacklist_mx ( self , mx ) : blacklisted = self . config . get ( 'blacklistes.mx' ) if not blacklisted : return False if mx in blacklisted : return True return False


def is_whitelist_mx(self, mx):
        """MX is whitelisted."""
        whitelistes = self.config.get('whitelistes.mx')
        if not whitelistes:
            return False
        if mx in whitelistes:
            return True
        return False

def is_whitelist_mx ( self , mx ) : whitelistes = self . config . get ( 'whitelistes.mx' ) if not whitelistes : return False if mx in whitelistes : return True return False


def internal_domains(self):
        """Get internal hosts from configuration."""
        domains = self.config.get('internal_domains')
        return domains if domains else []

def internal_domains ( self ) : domains = self . config . get ( 'internal_domains' ) return domains if domains else [ ]


def emitter_reputation(self, mx):
        """Return features about emitter."""
        if self.is_blacklist_mx(mx):
            return 'blacklisted'
        if self.is_whitelist_mx(mx):
            return 'whitelisted'
        return 'unknown'

def emitter_reputation ( self , mx ) : if self . is_blacklist_mx ( mx ) : return 'blacklisted' if self . is_whitelist_mx ( mx ) : return 'whitelisted' return 'unknown'


def emitter_certificate(self):
        """Get the certificate from emitter."""
        return None

def emitter_certificate ( self ) : return None


def mail_agent(self):
        """Get the mailer used for this message."""
        # XXX normalize better and more ?
        return self.message.mail.get('X-Mailer', '').lower()

def
mail_agent
(
self
)
:
# XXX normalize better and more ?
return
self
.
message
.
mail
.
get
(
'X-Mailer'
,
''
)
.
lower
(
)


def transport_signature(self):
        """Get the transport signature if any."""
        return self.message.mail.get('DKIM-Signature')

def transport_signature ( self ) : return self . message . mail . get ( 'DKIM-Signature' )


Return a global spam_score and related features .

def spam_informations(self):
        """Return a global spam_score and related features."""
        spam = SpamScorer(self.message.mail)
        return {'spam_score': spam.score,
                'spam_method': spam.method,
                'is_spam': spam.is_spam}

def spam_informations ( self ) : spam = SpamScorer ( self . message . mail ) return { 'spam_score' : spam . score , 'spam_method' : spam . method , 'is_spam' : spam . is_spam }


Return true if it s an internal message .

def is_internal(self):
        """Return true if it's an internal message."""
        from_ = self.message.mail.get('From')
        for domain in self.internal_domains:
            if domain in from_:
                return True
        return False

def is_internal ( self ) : from_ = self . message . mail . get ( 'From' ) for domain in self . internal_domains : if domain in from_ : return True return False


def get_signature_informations(self):
        """Get message signature features."""
        signed_parts = [x for x in self.message.attachments
                        if 'pgp-sign' in x.content_type]
        if not signed_parts:
            return {}
        sign = pgpy.PGPSignature()
        features = {'message_signed': True,
                    'message_signature_type': 'PGP'}
        try:
            sign.parse(signed_parts[0].data)
            features.update({'message_signer': sign.signer})
        except Exception as exc:
            log.error('Unable to parse pgp signature {}'.format(exc))
        return features

def
get_signature_informations
(
self
)
:
signed_parts
=
[
x
for
x
in
self
.
message
.
attachments
if
'pgp-sign'
in
x
.
content_type
]
if
not
signed_parts
:
return
{
}
sign
=
pgpy
.
PGPSignature
(
)
features
=
{
'message_signed'
:
True
,
'message_signature_type'
:
'PGP'
}
try
:
sign
.
parse
(
signed_parts
[
0
]
.
data
)
features
.
update
(
{
'message_signer'
:
sign
.
signer
}
)
except
Exception
as
exc
:
log
.
error
(
'Unable to parse pgp signature {}'
.
format
(
exc
)
)
return
features


def get_encryption_informations(self):
        """Get message encryption features."""
        is_encrypted = False
        if 'encrypted' in self.message.extra_parameters:
            is_encrypted = True

        # Maybe pgp/inline ?
        if not is_encrypted:
            try:
                body = self.message.body_plain.decode('utf-8')
                if body.startswith(PGP_MESSAGE_HEADER):
                    is_encrypted = True
            except UnicodeDecodeError:
                log.warn('Invalid body_plain encoding for message')
                pass

        return {'message_encrypted': is_encrypted,
                'message_encryption_method': 'pgp' if is_encrypted else ''}


def _get_features(self):
        """Extract privacy features."""
        features = self._features.copy()
        received = self.message.headers.get('Received', [])
        features.update(get_ingress_features(received, self.internal_domains))
        mx = features.get('ingress_server')
        reputation = None if not mx else self.emitter_reputation(mx)
        features['mail_emitter_mx_reputation'] = reputation
        features['mail_emitter_certificate'] = self.emitter_certificate()
        features['mail_agent'] = self.mail_agent
        features['is_internal'] = self.is_internal
        features.update(self.get_signature_informations())
        features.update(self.get_encryption_informations())
        features.update(self.spam_informations)

        if self.transport_signature:
            features.update({'transport_signed': True})
        return features

def _get_features ( self ) : features = self . _features . copy ( ) received = self . message . headers . get ( 'Received' , [ ] ) features . update ( get_ingress_features ( received , self . internal_domains ) ) mx = features . get ( 'ingress_server' ) reputation = None if not mx else self . emitter_reputation ( mx ) features [ 'mail_emitter_mx_reputation' ] = reputation features [ 'mail_emitter_certificate' ] = self . emitter_certificate ( ) features [ 'mail_agent' ] = self . mail_agent features [ 'is_internal' ] = self . is_internal features . update ( self . get_signature_informations ( ) ) features . update ( self . get_encryption_informations ( ) ) features . update ( self . spam_informations ) if self . transport_signature : features . update ( { 'transport_signed' : True } ) return features


def _compute_pi(self, participants, features):
        """Compute Privacy Indexes for a message."""
        log.info('PI features {}'.format(features))
        pi_cx = {}   # Contextual privacy index
        pi_co = {}   # Comportemental privacy index
        pi_t = {}   # Technical privacy index
        reput = features.get('mail_emitter_mx_reputation')
        if reput == 'whitelisted':
            pi_cx['reputation_whitelist'] = 20
        elif reput == 'unknown':
            pi_cx['reputation_unknow'] = 10
        known_contacts = []
        known_public_key = 0
        for part, contact in participants:
            if contact:
                known_contacts.append(contact)
                if contact.public_key:
                    known_public_key += 1
        if len(participants) == len(known_contacts):
            # - Si tous les contacts sont déjà connus le PIᶜˣ
            # augmente de la valeur du PIᶜᵒ le plus bas des PIᶜᵒ des contacts.
            contact_pi_cos = [x.pi['comportment'] for x in known_contacts
                              if x.pi and 'comportment' in x.pi]
            if contact_pi_cos:
                pi_cx['known_contacts'] = min(contact_pi_cos)

            if known_public_key == len(known_contacts):
                pi_co['contact_pubkey'] = 20
        ext_hops = features.get('nb_external_hops', 0)
        if ext_hops <= 1:
            tls = features.get('ingress_socket_version')
            if tls:
                if tls not in TLS_VERSION_PI:
                    log.warn('Unknown TLS version {}'.format(tls))
                else:
                    pi_t += TLS_VERSION_PI[tls]
        if features.get('mail_emitter_certificate'):
            pi_t['emitter_certificate'] = 10
        if features.get('transport_signed'):
            pi_t['transport_signed'] = 10
        if features.get('message_encrypted'):
            pi_t['encrypted'] = 30
        log.info('PI compute t:{} cx:{} co:{}'.format(pi_t, pi_cx, pi_co))
        return PIParameter({'technic': sum(pi_t.values()),
                            'context': sum(pi_cx.values()),
                            'comportment': sum(pi_co.values()),
                            'version': 0})


Process the message for privacy features and PI compute .

def process(self, user, message, participants):
        """
        Process the message for privacy features and PI compute.

        :param user: user the message belong to
        :ptype user: caliopen_main.user.core.User
        :param message: a message parameter that will be updated with PI
        :ptype message: NewMessage
        :param participants: an array of participant with related Contact
        :ptype participants: list(Participant, Contact)
        """
        features = self._get_features()
        message.pi = self._compute_pi(participants, features)
        il = compute_importance(user, message, features, participants)
        message.privacy_features = features
        message.importance_level = il

def process ( self , user , message , participants ) : features = self . _get_features ( ) message . pi = self . _compute_pi ( participants , features ) il = compute_importance ( user , message , features , participants ) message . privacy_features = features message . importance_level = il


def pstdev(avg, data):
    """Population standard deviation."""
    ss = sum((x - avg) ** 2 for x in data)
    return (ss / float(len(data))) ** 0.5

def pstdev ( avg , data ) : ss = sum ( ( x - avg ) ** 2 for x in data ) return ( ss / float ( len ( data ) ) ) ** 0.5


Get an histogram for a contact email and compute basic infos .

def _compute_histogram(self, email):
        """Get an histogram for a contact email and compute basic infos."""
        histo = ParticipantHistogram(self.user, 'day')
        data = histo.find_by_address(email)
        log.info('Got email {0} histogram result {1}'.format(email, data))
        if data:
            values = [x[1] for x in data]
            total = sum(values)
            avg = total / float(len(values))
            return (total, avg, pstdev(avg, values))
        return (0.0, 0.0, 0.0)

def
_compute_histogram
(
self
,
email
)
:
histo
=
ParticipantHistogram
(
self
.
user
,
'day'
)
data
=
histo
.
find_by_address
(
email
)
log
.
info
(
'Got email {0} histogram result {1}'
.
format
(
email
,
data
)
)
if
data
:
values
=
[
x
[
1
]
for
x
in
data
]
total
=
sum
(
values
)
avg
=
total
/
float
(
len
(
values
)
)
return
(
total
,
avg
,
pstdev
(
avg
,
values
)
)
return
(
0.0
,
0.0
,
0.0
)


Get privacy features related to message histograms .

def _get_histogram(self, contact):
        """Get privacy features related to message histograms."""
        histograms = [self._compute_histogram(x.address)
                      for x in contact.emails]
        if histograms:
            best = max(histograms)
            return {'message_day_total': best[0],
                    'message_day_avg': best[1],
                    'message_day_pstdev': best[2]}
        return {}

def _get_histogram ( self , contact ) : histograms = [ self . _compute_histogram ( x . address ) for x in contact . emails ] if histograms : best = max ( histograms ) return { 'message_day_total' : best [ 0 ] , 'message_day_avg' : best [ 1 ] , 'message_day_pstdev' : best [ 2 ] } return { }


def _get_technical(self, contact):
        """Get technical features for a contact."""
        features = {}
        if hasattr(contact, 'public_keys') and contact.public_keys:
            max_size = max([x.size for x in contact.public_keys])
            if max_size:
                features.update({'public_key_best_size': max_size})
        return features

def _get_technical ( self , contact ) : features = { } if hasattr ( contact , 'public_keys' ) and contact . public_keys : max_size = max ( [ x . size for x in contact . public_keys ] ) if max_size : features . update ( { 'public_key_best_size' : max_size } ) return features


Process a contact to extract all privacy features .

def process(self, contact):
        """Process a contact to extract all privacy features."""
        features = self._get_histogram(contact)
        features.update(self._get_technical(contact))
        log.info('Contact {0} have features {1}'.
                 format(contact.contact_id, features))
        pi = self._compute_pi(contact, features)
        return pi, marshal_features(features)

def
process
(
self
,
contact
)
:
features
=
self
.
_get_histogram
(
contact
)
features
.
update
(
self
.
_get_technical
(
contact
)
)
log
.
info
(
'Contact {0} have features {1}'
.
format
(
contact
.
contact_id
,
features
)
)
pi
=
self
.
_compute_pi
(
contact
,
features
)
return
pi
,
marshal_features
(
features
)


Check that a feature fit between min and max value if apply .

def check_feature_bounding(value, feature):
    """Check that a feature fit between min and max value if apply."""
    if 'min' in feature and value < feature['min']:
        return False
    if 'max' in feature and value > feature['max']:
        return False
    return True

def check_feature_bounding ( value , feature ) : if 'min' in feature and value < feature [ 'min' ] : return False if 'max' in feature and value > feature [ 'max' ] : return False return True


def find_feature(name):
    """Find a feature by its name."""
    if name in FEATURES:
        return FEATURES[name]

def find_feature ( name ) : if name in FEATURES : return FEATURES [ name ]


Check that a feature have a correct value .

def check_feature(name, value):
    """Check that a feature have a correct value."""

    feature = find_feature(name)
    if not feature:
        return False

    if feature['type'] == 'int':
        try:
            int(value)
        except ValueError:
            return False
        return check_feature_bounding(value, feature)

    elif feature['type'] == 'bool':
        return True if value in (True, False) else False
    elif feature['type'] == 'string':
        return True
    log.error('Invalid feature type %s, fail silently for the moment' %
              feature['type'])


Unmarshal a feature to it s type representation .

def unmarshal_feature(name, value):
    """Unmarshal a feature to it's type representation."""
    if not check_feature(name, value):
        raise ValueError('Invalid value {} for feature {}'.format(value, name))
    feature = find_feature(name)
    if not feature:
        log.warn('Unknow feature %s' % name)
        return None
    if feature['type'] == 'int':
        return int(value)
    elif feature['type'] == 'bool':
        return True if value.lower().startswith('t') else False
    elif feature['type'] == 'string':
        return value

def
unmarshal_feature
(
name
,
value
)
:
if
not
check_feature
(
name
,
value
)
:
raise
ValueError
(
'Invalid value {} for feature {}'
.
format
(
value
,
name
)
)
feature
=
find_feature
(
name
)
if
not
feature
:
log
.
warn
(
'Unknow feature %s'
%
name
)
return
if
feature
[
'type'
]
==
'int'
:
return
int
(
value
)
elif
feature
[
'type'
]
==
'bool'
:
return
True
if
value
.
lower
(
)
.
startswith
(
't'
)
else
False
elif
feature
[
'type'
]
==
'string'
:
return
value


Unmarshal a dict of features for suitable output .

def unmarshal_features(features):
    """Unmarshal a dict of features for suitable output."""
    res = {}
    for name, value in features.items():
        try:
            new_value = unmarshal_feature(name, value)
            res[name] = new_value
        except ValueError:
            log.warn('Feature {} with {} do not marshall'.format(name, value))
            pass
    return res

def
unmarshal_features
(
features
)
:
res
=
{
}
for
name
,
value
in
features
.
items
(
)
:
try
:
new_value
=
unmarshal_feature
(
name
,
value
)
res
[
name
]
=
new_value
except
ValueError
:
log
.
warn
(
'Feature {} with {} do not marshall'
.
format
(
name
,
value
)
)
pass
return
res


Unmarshall a dict of features suitable for storage .

def marshal_features(features):
    """Unmarshall a dict of features suitable for storage."""
    res = {}
    for k, v in features.items():
        if not (v == '' or v is None):
            res[k] = str(v)
    return res

def marshal_features ( features ) : res = { } for k , v in features . items ( ) : if not ( v == '' or v is None ) : res [ k ] = str ( v ) return res


Initialize privacy features for a given object type .

def init_features(type_):
    """Initialize privacy features for a given object type."""
    if type_ == 'contact':
        features = CONTACT
    elif type_ == 'device':
        features = DEVICE
    elif type_ == 'message':
        features = MESSAGE
    else:
        raise ValueError('Invalid feature type %s' % type_)

    output = {}
    for k, v in features.items():
        if v['type'] == 'bool':
            value = False
        elif v['type'] == 'int':
            value = 0
        else:
            value = ''
        output.update({k: value})
    return output

def
init_features
(
type_
)
:
if
type_
==
'contact'
:
features
=
CONTACT
elif
type_
==
'device'
:
features
=
DEVICE
elif
type_
==
'message'
:
features
=
MESSAGE
else
:
raise
ValueError
(
'Invalid feature type %s'
%
type_
)
output
=
{
}
for
k
,
v
in
features
.
items
(
)
:
if
v
[
'type'
]
==
'bool'
:
value
=
False
elif
v
[
'type'
]
==
'int'
:
value
=
0
else
:
value
=
''
output
.
update
(
{
k
:
value
}
)
return
output


Normalize tls information from a Received header .

def normalize_ssocket_infos(socket_info, cipher):
    """Normalize tls information from a Received header."""
    if socket_info:
        tls_version = socket_info.replace('_', '.').lower()
    else:
        tls_version = ''
    return tls_version, cipher

def normalize_ssocket_infos ( socket_info , cipher ) : if socket_info : tls_version = socket_info . replace ( '_' , '.' ) . lower ( ) else : tls_version = '' return tls_version , cipher


Try to find information about ingress server that send this mail .

def get_ingress_features(headers, internal_domains=None):
    """Try to find information about ingress server that send this mail."""
    internal_domains = internal_domains if internal_domains else []

    def get_host(line):
        if ' ' not in line:
            return None
        parts = line.split(' ')
        return parts[0]

    def parse_ingress(header):
        """Parse detected ingress header to find encryption infos."""
        header = header.replace('\n', '')
        # XXX to complete but MUST match with tls_version, cipher groups
        searches = ['.*using (\S+) with cipher ([\S-]+)']
        for regex in searches:
            search = re.compile(regex, re.MULTILINE)
            match = search.match(header)
            if match:
                return match.groups()
        return None

    found_features = {}

    # First step Search for 'paths' (from / by) information
    search = re.compile(r'^from (.*) by (.*)', re.MULTILINE + re.DOTALL)
    paths = []
    for r in headers:
        r = r.replace('\n', '')
        match = search.match(r)
        if match:
            groups = match.groups()
            from_ = get_host(groups[0])
            by = get_host(groups[1])
            if from_ and by:
                paths.append((from_, by, groups))
            else:
                log.debug('Invalid from {} or by {} path in {}'.
                          format(from_, by, r))
        else:
            if r.startswith('by'):
                # XXX first hop, to consider ?
                pass
            else:
                log.debug('Received header, does not match format {}'.
                          format(r))

    # Second step: qualify path if internal and try to find the ingress one
    ingress = None
    internal_hops = 0
    for path in paths:
        is_internal = False
        for internal in internal_domains:
            if internal in path[0]:
                is_internal = True
            else:
                if internal in path[1]:
                    ingress = path
                    break
        if is_internal:
            internal_hops += 1

    # Qualify ingress connection
    if ingress:
        cnx_info = parse_ingress(ingress[2][0])
        if cnx_info and len(cnx_info) > 1:
            tls_version, cipher = normalize_ssocket_infos(cnx_info)
            found_features.update({'ingress_socket_version': tls_version,
                                   'ingress_cipher': cipher})
        found_features.update({'ingress_server': ingress[0]})

    # Try to count external hops
    external_hops = len(paths) - internal_hops
    found_features.update({'nb_external_hops': external_hops})
    return found_features


Get the max importance level from tags .

def get_importance_tags(tags):
    """Get the max importance level from tags."""
    max_value = 0
    for tag in tags:
        if getattr(tag, 'importance_level'):
            if tag.importance_level > max_value:
                max_value = tag.importance_level
    return max_value

def get_importance_tags ( tags ) : max_value = 0 for tag in tags : if getattr ( tag , 'importance_level' ) : if tag . importance_level > max_value : max_value = tag . importance_level return max_value


Compute importance level for an inbound message .

def compute_importance(user, message, features, participants):
    """Compute importance level for an inbound message."""
    positive = 0
    negative = 0
    scores = []
    # Spam level
    if features.get('is_spam'):
        spam_score = features.get('spam_score', 0) / 100.0
        negative -= spam_score * abs(MIN_VALUE * SPAM_RATIO)
        scores.append(('spam_score', spam_score))
    # PI message
    if message.pi.context:
        context_score = message.pi.context / 100.0
        positive += context_score * PI_CX_RATIO * MAX_VALUE
        scores.append(('context_score', context_score))
    if message.pi.comportment:
        comportment_score = message.pi.comportment / 100.0
        positive += comportment_score * PI_CO_RATIO * MAX_VALUE
        scores.append(('comportment_score', comportment_score))
    # Tags
    if message.tags:
        tag_score = get_importance_tags(message.tags)
        positive += tag_score / MAX_TAG_SCORE * CONTEXT_RATIO
        scores.append(('tag_score', tag_score))

    # It's a reply ?
    if message.external_references:
        # XXX find if we know related messages to increment level
        positive += REPLY_VALUE
        scores.append(('reply_score', REPLY_VALUE))

    log.info('Importance scores: {0}'.format(scores))
    # Return the final value
    if negative:
        return round(max(MIN_VALUE, negative))
    return round(min(MAX_VALUE, positive))


Instanciate a date histrogram for an user .

def __init__(self, user, resolution='day'):
        """Instanciate a date histrogram for an user."""
        self.esclient = get_index_connection()
        self.user = user
        self.resolution = resolution

def __init__ ( self , user , resolution = 'day' ) : self . esclient = get_index_connection ( ) self . user = user self . resolution = resolution


def _format_results(self, results):
        """Return a list of date, value tuples."""
        return [(parse_date(x['key_as_string']),
                 x['doc_count']) for x in results]

def _format_results ( self , results ) : return [ ( parse_date ( x [ 'key_as_string' ] ) , x [ 'doc_count' ] ) for x in results ]


Build a date histogram with a participant address .

def find_by_address(self, address):
        """Build a date histogram with a participant address."""
        return self._do_query(address, term='participants.address')

def find_by_address ( self , address ) : return self . _do_query ( address , term = 'participants.address' )


Build a date histogram with a know contact .

def find_by_contact_id(self, contact_id):
        """Build a date histogram with a know contact."""
        return self._do_query(contact_id, term='participants.contact_id')

def find_by_contact_id ( self , contact_id ) : return self . _do_query ( contact_id , term = 'participants.contact_id' )


Extract different spam scores from mail .

def extract_mail_spam_scores(mail):
    """Extract different spam scores from mail."""
    scores = {}
    score = mail.get('X-Spam-Score', 0)
    if score:
        try:
            score = float(score)
            scores['score'] = score
        except TypeError:
            log.debug('Invalid type for X-Spam-Score value {}'.
                      format(score))
    level = mail.get('X-Spam-Level', '')
    if '*' in level:
        # SpamAssassin style, level is *** notation, up to 25 *
        scores['level'] = level.count('*')
    status = mail.get('X-Spam-Status', '')
    if status:
        match = re.match('^(.*), score=(\d.\d).*', status)
        if match:
            flag = match.groups()[0]
            scores['status_score'] = float(match.groups()[1])
            if flag.lower().startswith('y'):
                scores['flag'] = True
            else:
                scores['flag'] = False

    flag = mail.get('X-Spam-Flag', '')
    if flag.lower().startswith('y'):
        scores['flag'] = True

    return scores


Compute a global score using multiple methods and mail .

def __init__(self, mail):
        """Compute a global score using multiple methods and mail."""
        self.scores = extract_mail_spam_scores(mail)
        log.debug('Found spam scores {s}'.format(s=self.scores))
        if 'level' in self.scores:
            self.method = 'level'
            self.score = min(100.0, self.scores['level'] * 4.0)
        if 'score' in self.scores:
            self.method = 'score'
            self.score = min(0.0, self.scores['score']) * 10.0
        if 'status_score' in self.scores:
            self.method = 'status'
            if self.scores['status_score'] < 2.0:
                self.score = 0.0
            else:
                tmp_score = min(100.0, self.scores['status_score'] * 10)
                self.score = max(0.0, tmp_score)
        if self.scores.get('flag'):
            self.is_spam = True
            self.source_flag = self.scores['flag']
        if not self.is_spam and self.score >= 50:
            self.is_spam = True
        if self.score < 0:
            self.score = 0.0
        if self.score > 100:
            self.score = 100.0

def
__init__
(
self
,
mail
)
:
self
.
scores
=
extract_mail_spam_scores
(
mail
)
log
.
debug
(
'Found spam scores {s}'
.
format
(
s
=
self
.
scores
)
)
if
'level'
in
self
.
scores
:
self
.
method
=
'level'
self
.
score
=
min
(
100.0
,
self
.
scores
[
'level'
]
*
4.0
)
if
'score'
in
self
.
scores
:
self
.
method
=
'score'
self
.
score
=
min
(
0.0
,
self
.
scores
[
'score'
]
)
*
10.0
if
'status_score'
in
self
.
scores
:
self
.
method
=
'status'
if
self
.
scores
[
'status_score'
]
<
2.0
:
self
.
score
=
0.0
else
:
tmp_score
=
min
(
100.0
,
self
.
scores
[
'status_score'
]
*
10
)
self
.
score
=
max
(
0.0
,
tmp_score
)
if
self
.
scores
.
get
(
'flag'
)
:
self
.
is_spam
=
True
self
.
source_flag
=
self
.
scores
[
'flag'
]
if
not
self
.
is_spam
and
self
.
score
>=
50
:
self
.
is_spam
=
True
if
self
.
score
<
0
:
self
.
score
=
0.0
if
self
.
score
>
100
:
self
.
score
=
100.0


Create a new instance of an user device qualifier .

def __init__(self, user):
        """Create a new instance of an user device qualifier."""
        self.user = user
        self.conf = Configuration('global').configuration

def __init__ ( self , user ) : self . user = user self . conf = Configuration ( 'global' ) . configuration


def process(self, device):
        """Process a device to qualify it."""
        extractor = DeviceFeature(self.user, self.conf)
        pi, features = extractor.process(device)
        device.privacy_features = marshal_features(features)
        device.pi = pi
        return True

def process ( self , device ) : extractor = DeviceFeature ( self . user , self . conf ) pi , features = extractor . process ( device ) device . privacy_features = marshal_features ( features ) device . pi = pi return True


Create a new instance of an user message qualifier .

def __init__(self, user, identity):
        """Create a new instance of an user message qualifier."""
        self.user = user
        self.identity = identity

def __init__ ( self , user , identity ) : self . user = user self . identity = identity


def _get_tags(self, message):
        """Get tags for a message."""
        tags = []
        if message.privacy_features.get('is_internal', False):
            # XXX do not hardcode the wanted tag
            internal_tag = [x for x in self.user.tags if x.name == 'internal']
            if internal_tag:
                tags.append(internal_tag[0].name)
        message.tags = tags

def
_get_tags
(
self
,
message
)
:
tags
=
[
]
if
message
.
privacy_features
.
get
(
'is_internal'
,
False
)
:
# XXX do not hardcode the wanted tag
internal_tag
=
[
x
for
x
in
self
.
user
.
tags
if
x
.
name
==
'internal'
]
if
internal_tag
:
tags
.
append
(
internal_tag
[
0
]
.
name
)
message
.
tags
=
tags


Process lookup sequence to find discussion to associate .

def lookup(self, sequence):
        """Process lookup sequence to find discussion to associate."""
        log.debug('Lookup sequence %r' % sequence)
        for prop in sequence:
            try:
                kls = self._lookups[prop[0]]
                log.debug('Will lookup %s with value %s' %
                          (prop[0], prop[1]))
                return kls.get(self.user, prop[1])
            except NotFound:
                log.debug('Lookup type %s with value %s failed' %
                          (prop[0], prop[1]))

        return None

def
lookup
(
self
,
sequence
)
:
log
.
debug
(
'Lookup sequence %r'
%
sequence
)
for
prop
in
sequence
:
try
:
kls
=
self
.
_lookups
[
prop
[
0
]
]
log
.
debug
(
'Will lookup %s with value %s'
%
(
prop
[
0
]
,
prop
[
1
]
)
)
return
kls
.
get
(
self
.
user
,
prop
[
1
]
)
except
NotFound
:
log
.
debug
(
'Lookup type %s with value %s failed'
%
(
prop
[
0
]
,
prop
[
1
]
)
)
return


Initialize lookup classes for the related sequence .

def create_lookups(self, sequence, message):
        """Initialize lookup classes for the related sequence."""
        for prop in sequence:
            kls = self._lookups[prop[0]]
            params = {
                kls._pkey_name: prop[1],
                'discussion_id': message.discussion_id
            }
            lookup = kls.create(self.user, **params)
            log.info('Create lookup %r' % lookup)

def
create_lookups
(
self
,
sequence
,
message
)
:
for
prop
in
sequence
:
kls
=
self
.
_lookups
[
prop
[
0
]
]
params
=
{
kls
.
_pkey_name
:
prop
[
1
]
,
'discussion_id'
:
message
.
discussion_id
}
lookup
=
kls
.
create
(
self
.
user
,
*
*
params
)
log
.
info
(
'Create lookup %r'
%
lookup
)


Try to find a related contact and return a Participant instance .

def get_participant(self, message, participant):
        """Try to find a related contact and return a Participant instance."""
        p = Participant()
        p.address = participant.address.lower()
        p.type = participant.type
        p.label = participant.label
        p.protocol = message.message_protocol
        log.debug('Will lookup contact {} for user {}'.
                  format(participant.address, self.user.user_id))
        try:
            c = Contact.lookup(self.user, participant.address)
        except Exception as exc:
            log.error("Contact lookup failed in get_participant for participant {} : {}".format(vars(participant), exc))
            raise exc
        if c:
            p.contact_ids = [c.contact_id]
        else:
            if p.address == self.identity.identifier and self.user.contact_id:
                p.contact_ids = [self.user.contact_id]
        return p, c


Return list of lookups ( type value ) from a mail message and the first lookup hash from that list

def lookup_discussion_sequence(self, mail, message):
        """
        Return list of lookups (type, value) from a mail message
        and the first lookup'hash from that list
        """
        seq = []

        # lists lookup first
        lists = []
        for list_id in mail.extra_parameters.get('lists', []):
            lists.append(list_id)
        if len(lists) > 0:
            # list_ids are considered `participants`
            # build uris' hash and upsert lookup tables
            participants = []
            for list_id in lists:
                participant = Participant()
                participant.address = list_id.lower()
                participant.protocol = 'email'
                participant.type = 'list-id'
                participants.append(participant)
                discuss = Discussion(self.user)
                discuss.upsert_lookups_for_participants(participants)
                # add list-id as a participant to the message
                message.participants.append(participant)
            hash = hash_participants_uri(participants)
            seq.append(('list', hash['hash']))

        # then participants
        seq.append(('hash', message.hash_participants))

        # try to link message to external thread's root message-id
        #        if len(message.external_references["ancestors_ids"]) > 0:
        #            seq.append(("thread",
        #                        message.external_references["ancestors_ids"][0]))
        #        elif message.external_references["parent_id"]:
        #            seq.append(("thread", message.external_references["parent_id"]))
        #        elif message.external_references["message_id"]:
        #            seq.append(("thread", message.external_references["message_id"]))

        return seq, seq[0][1]


def process_inbound(self, raw):
        """Process inbound message.

        @param raw: a RawMessage object
        @rtype: NewMessage
        """
        email = MailMessage(raw.raw_data)
        new_message = NewInboundMessage()
        new_message.raw_msg_id = raw.raw_msg_id
        new_message.subject = email.subject
        new_message.body_html = email.body_html
        new_message.body_plain = email.body_plain
        new_message.date = email.date
        new_message.size = email.size
        new_message.protocol = email.message_protocol
        new_message.is_unread = True
        new_message.is_draft = False
        new_message.is_answered = False
        new_message.is_received = True
        new_message.importance_level = 0  # XXX tofix on parser
        new_message.external_references = email.external_references

        participants = []
        for p in email.participants:
            p.address = p.address.lower()
            try:
                participant, contact = self.get_participant(email, p)
                new_message.participants.append(participant)
                participants.append((participant, contact))
            except Exception as exc:
                log.error("process_inbound failed to lookup participant for email {} : {}".format(vars(email), exc))
                raise exc

        if not participants:
            raise Exception("no participant found in raw email {}".format(
                raw.raw_msg_id))

        for a in email.attachments:
            attachment = Attachment()
            attachment.content_type = a.content_type
            attachment.file_name = a.filename
            attachment.size = a.size
            attachment.mime_boundary = a.mime_boundary
            if hasattr(a, "is_inline"):
                attachment.is_inline = a.is_inline
            new_message.attachments.append(attachment)

        # Compute PI !!
        conf = Configuration('global').configuration
        extractor = InboundMailFeature(email, conf)
        extractor.process(self.user, new_message, participants)

        # compute user tags
        self._get_tags(new_message)
        # embed external flags if any
        new_message.ext_tags = email.external_flags
        if new_message.tags:
            log.debug('Resolved tags {}'.format(new_message.tags))

        # build discussion_id from lookup_sequence
        lookup_sequence, discussion_id = self.lookup_discussion_sequence(email,
                                                                         new_message)
        log.debug('Lookup with sequence {} gives {}'.format(lookup_sequence,
                                                            discussion_id))
        new_message.discussion_id = discussion_id

        # upsert lookup tables
        discuss = Discussion(self.user)
        discuss.upsert_lookups_for_participants(new_message.participants)
        # Format features
        new_message.privacy_features = \
            marshal_features(new_message.privacy_features)
        try:
            new_message.validate()
        except Exception as exc:
            log.error(
                "validation failed with error : « {} » \
                for new_message {}[dump : {}]".format(
                    exc, new_message, vars(new_message)))
            raise exc

        return new_message


Return list of lookup type value from a tweet .

def lookup_discussion_sequence(self, message, *args, **kwargs):
        """Return list of lookup type, value from a tweet."""
        seq = list()

        participants = message.hash_participants
        seq.append(('hash', participants))

        return seq, seq[0][1]

def lookup_discussion_sequence ( self , message , * args , * * kwargs ) : seq = list ( ) participants = message . hash_participants seq . append ( ( 'hash' , participants ) ) return seq , seq [ 0 ] [ 1 ]


def process_inbound(self, raw):
        """
        Process inbound message.

        @param raw: a RawMessage object
            which should be a json conforming to
            https://developer.twitter.com/en/docs/direct-messages/\
            sending-and-receiving/guides/message-create-object
        @rtype: NewMessage
        """
        tweet = TwitterDM(raw.raw_data)
        new_message = NewInboundMessage()
        new_message.raw_msg_id = raw.raw_msg_id
        new_message.body_plain = tweet.body_plain
        new_message.date = tweet.date
        new_message.protocol = tweet.protocol
        new_message.is_unread = True
        new_message.is_draft = False
        new_message.is_answered = False
        new_message.is_received = True
        new_message.importance_level = 0  # XXX tofix on parser
        new_message.external_references = tweet.external_references

        participants = []
        for p in tweet.participants:
            p.address = clean_twitter_address(p.address)
            participant, contact = self.get_participant(tweet, p)
            new_message.participants.append(participant)
            participants.append((participant, contact))

        if not participants:
            raise Exception("no participant found in raw tweet {}".format(
                raw.raw_msg_id))

        # Compute PI !!
        # TODO

        # compute tags
        self._get_tags(new_message)
        if new_message.tags:
            log.debug('Resolved tags {}'.format(new_message.tags))

        # build discussion_id from lookup_sequence
        lookup_sequence, discussion_id = self.lookup_discussion_sequence(
            new_message)
        log.debug('Lookup with sequence {} gives {}'.format(lookup_sequence,
                                                            discussion_id))
        new_message.discussion_id = discussion_id

        # upsert lookup tables
        discuss = Discussion(self.user)
        discuss.upsert_lookups_for_participants(new_message.participants)
        # Format features
        new_message.privacy_features = \
            marshal_features(new_message.privacy_features)
        try:
            new_message.validate()
        except Exception as exc:
            log.error(
                "validation failed with error : « {} » \
                for new_message {}[dump : {}]".format(
                    exc, new_message, vars(new_message)))
            raise exc

        return new_message


def process_inbound(self, raw):
        """
        Process inbound message.

        @param raw: a RawMessage object
            which should be a json conforming to
            https://docs.joinmastodon.org/api/entities/#status
        @rtype: NewMessage
        """
        toot = MastodonStatus(raw.raw_data)
        new_message = NewInboundMessage()
        new_message.raw_msg_id = raw.raw_msg_id
        new_message.body_html = toot.body_html
        new_message.date = toot.date
        new_message.protocol = toot.protocol
        new_message.is_unread = True
        new_message.is_draft = False
        new_message.is_answered = False
        new_message.is_received = True
        new_message.importance_level = 0  # XXX tofix on parser
        new_message.external_references = toot.external_references

        participants = []
        for p in toot.participants:
            p.address = p.address
            participant, contact = self.get_participant(toot, p)
            new_message.participants.append(participant)
            participants.append((participant, contact))

        if not participants:
            raise Exception("no participant found in raw tweet {}".format(
                raw.raw_msg_id))

        # Compute PI !!
        # TODO

        # compute tags
        self._get_tags(new_message)
        if new_message.tags:
            log.debug('Resolved tags {}'.format(new_message.tags))

        # build discussion_id from lookup_sequence
        lookup_sequence, discussion_id = self.lookup_discussion_sequence(
            new_message)
        log.debug('Lookup with sequence {} gives {}'.format(lookup_sequence,
                                                            discussion_id))
        new_message.discussion_id = discussion_id

        # upsert lookup tables
        discuss = Discussion(self.user)
        discuss.upsert_lookups_for_participants(new_message.participants)
        # Format features
        new_message.privacy_features = \
            marshal_features(new_message.privacy_features)
        try:
            new_message.validate()
        except Exception as exc:
            log.error(
                "validation failed with error : « {} » \
                for new_message {}[dump : {}]".format(
                    exc, new_message, vars(new_message)))
            raise exc

        return new_message


def process(self, contact):
        """Qualification for a contact."""
        extractor = ContactFeature(self.user)
        pi, features = extractor.process(contact)
        # XXX for the moment apply features and pi
        if not contact.pi:
            current_pi = None
        else:
            current_pi = contact.pi.marshall_dict()
        new_pi = pi.serialize()
        new_pi.pop('date_update')
        current = {}
        patch = {}
        if pi.technic or pi.context or pi.comportment:
            current.update({'pi': current_pi})
            patch.update({'pi': new_pi})
        if contact.privacy_features:
            current_features = contact.privacy_features
        else:
            current_features = {}
        if current_features != features:
            current.update({'privacy_features': contact.privacy_features})
            patch.update({'privacy_features': features})
        if current and patch:
            patch.update({'current_state': current})
            log.info('Will patch with {0}'.format(patch))
            contact.apply_patch(patch, db=True)
            return True
        log.debug('No confidentiality update for contact')
        return False


Create a new instance of an contact email qualifier .

def __init__(self, user):
        """Create a new instance of an contact email qualifier."""
        self.user = user
        conf = Configuration('global').configuration
        self.key_disco = PublicKeyDiscoverer(conf)

def __init__ ( self , user ) : self . user = user conf = Configuration ( 'global' ) . configuration self . key_disco = PublicKeyDiscoverer ( conf )


def create_new_email(self, contact, email):
        """Add a new email for a contact."""
        found_keys = self.key_disco.search_email(email)
        if found_keys:
            log.info('Found keys for email {0}: {1}'.format(email, found_keys))
            self._process_new_keys(contact, found_keys)

def
create_new_email
(
self
,
contact
,
email
)
:
found_keys
=
self
.
key_disco
.
search_email
(
email
)
if
found_keys
:
log
.
info
(
'Found keys for email {0}: {1}'
.
format
(
email
,
found_keys
)
)
self
.
_process_new_keys
(
contact
,
found_keys
)


Create a new inbound messsage handler from a nats connection .

def __init__(self, nats_cnx):
        """Create a new inbound messsage handler from a nats connection."""
        self.natsConn = nats_cnx

def __init__ ( self , nats_cnx ) : self . natsConn = nats_cnx


def process_raw(self, msg, payload):
        """Process an inbound raw message."""
        nats_error = {
            'error': '',
            'message': 'inbound email message process failed'
        }
        nats_success = {
            'message': 'OK : inbound email message proceeded'
        }
        try:
            user = User.get(payload['user_id'])
            identity = UserIdentity.get(user, payload['identity_id'])
            deliver = UserMailDelivery(user, identity)
            new_message = deliver.process_raw(payload['message_id'])
            nats_success['message_id'] = str(new_message.message_id)
            nats_success['discussion_id'] = str(new_message.discussion_id)
            self.natsConn.publish(msg.reply, json.dumps(nats_success))
        except DuplicateObject:
            log.info("Message already imported : {}".format(payload))
            nats_success['message_id'] = str(payload['message_id'])
            nats_success['discussion_id'] = ""  # message has not been parsed
            nats_success['message'] = 'raw message already imported'
            self.natsConn.publish(msg.reply, json.dumps(nats_success))
        except Exception as exc:
            # TODO: handle abort exception and report it as special case
            exc_info = sys.exc_info()
            log.error("deliver process failed for raw {}: {}".
                      format(payload, traceback.print_exception(*exc_info)))
            nats_error['error'] = str(exc)
            self.natsConn.publish(msg.reply, json.dumps(nats_error))
            return exc


def handler(self, msg):
        """Handle an process_raw nats messages."""
        payload = json.loads(msg.data)
        log.info('Get payload order {}'.format(payload))
        if payload['order'] == "process_raw":
            self.process_raw(msg, payload)
        else:
            log.warn(
                'Unhandled payload order "{}" \
                (queue: SMTPqueue, subject : inboundSMTP)'.format(
                    payload['order']))
            raise NotImplementedError


def process_raw(self, msg, payload):
        """Process an inbound raw message."""
        nats_error = {
            'error': '',
            'message': 'inbound twitter message process failed'
        }
        nats_success = {
            'message': 'OK : inbound twitter message proceeded'
        }
        try:
            user = User.get(payload['user_id'])
            identity = UserIdentity.get(user, payload['identity_id'])
            deliver = UserTwitterDelivery(user, identity)
            new_message = deliver.process_raw(payload['message_id'])
            nats_success['message_id'] = str(new_message.message_id)
            nats_success['discussion_id'] = str(new_message.discussion_id)
            self.natsConn.publish(msg.reply, json.dumps(nats_success))
        except DuplicateObject:
            log.info("Message already imported : {}".format(payload))
            nats_success['message_id'] = str(payload['message_id'])
            nats_success['discussion_id'] = ""  # message has not been parsed
            nats_success['message'] = 'raw message already imported'
            self.natsConn.publish(msg.reply, json.dumps(nats_success))
        except Exception as exc:
            # TODO: handle abort exception and report it as special case
            exc_info = sys.exc_info()
            log.error("deliver process failed for raw {}: {}".
                      format(payload, traceback.print_exception(*exc_info)))
            nats_error['error'] = str(exc.message)
            self.natsConn.publish(msg.reply, json.dumps(nats_error))
            return exc


def handler(self, msg):
        """Handle an process_raw nats messages."""
        payload = json.loads(msg.data)
        log.info('Get payload order {}'.format(payload['order']))
        if payload['order'] == "process_raw":
            self.process_raw(msg, payload)
        else:
            log.warn('Unhandled payload type {}'.format(payload['order']))

def
handler
(
self
,
msg
)
:
payload
=
json
.
loads
(
msg
.
data
)
log
.
info
(
'Get payload order {}'
.
format
(
payload
[
'order'
]
)
)
if
payload
[
'order'
]
==
"process_raw"
:
self
.
process_raw
(
msg
,
payload
)
else
:
log
.
warn
(
'Unhandled payload type {}'
.
format
(
payload
[
'order'
]
)
)


def process_raw(self, msg, payload):
        """Process an inbound raw message."""
        nats_error = {
            'error': '',
            'message': 'inbound mastodon message process failed'
        }
        nats_success = {
            'message': 'OK : inbound mastodon message proceeded'
        }
        try:
            user = User.get(payload['user_id'])
            identity = UserIdentity.get(user, payload['identity_id'])
            deliver = UserMastodonDelivery(user, identity)
            new_message = deliver.process_raw(payload['message_id'])
            nats_success['message_id'] = str(new_message.message_id)
            nats_success['discussion_id'] = str(new_message.discussion_id)
            self.natsConn.publish(msg.reply, json.dumps(nats_success))
        except DuplicateObject:
            log.info("Message already imported : {}".format(payload))
            nats_success['message_id'] = str(payload['message_id'])
            nats_success['discussion_id'] = ""  # message has not been parsed
            nats_success['message'] = 'raw message already imported'
            self.natsConn.publish(msg.reply, json.dumps(nats_success))
        except Exception as exc:
            # TODO: handle abort exception and report it as special case
            exc_info = sys.exc_info()
            log.error("deliver process failed for raw {}: {}".
                      format(payload, traceback.print_exception(*exc_info)))
            nats_error['error'] = str(exc.message)
            self.natsConn.publish(msg.reply, json.dumps(nats_error))
            return exc


def process_update(self, msg, payload):
        """Process a contact update message."""
        # XXX validate payload structure
        if 'user_id' not in payload or 'contact_id' not in payload:
            raise Exception('Invalid contact_update structure')
        user = User.get(payload['user_id'])
        contact = Contact(user, contact_id=payload['contact_id'])
        contact.get_db()
        contact.unmarshall_db()
        qualifier = ContactMessageQualifier(user)
        log.info('Will process update for contact {0} of user {1}'.
                 format(contact.contact_id, user.user_id))
        # TODO: (re)discover GPG keys
        qualifier.process(contact)


def handler(self, msg):
        """Handle an process_raw nats messages."""
        payload = json.loads(msg.data)
        # log.info('Get payload order {}'.format(payload['order']))
        if payload['order'] == "contact_update":
            self.process_update(msg, payload)
        else:
            log.warn(
                'Unhandled payload order "{}" \
                (queue: contactQueue, subject : contactAction)'.format(
                    payload['order']))
            raise NotImplementedError


Discover public keys related to a new contact identifier .

def process_key_discovery(self, msg, payload):
        """Discover public keys related to a new contact identifier."""
        if 'user_id' not in payload or 'contact_id' not in payload:
            raise Exception('Invalid contact_update structure')
        user = User.get(payload['user_id'])
        contact = Contact(user.user_id, contact_id=payload['contact_id'])
        contact.get_db()
        contact.unmarshall_db()
        manager = ContactPublicKeyManager()
        founds = []
        for ident in payload.get('emails', []):
            log.info('Process email identity {0}'.format(ident['address']))
            discovery = manager.process_identity(user, contact,
                                                 ident['address'], 'email')
            if discovery:
                founds.append(discovery)
        for ident in payload.get('identities', []):
            log.info('Process identity {0}:{1}'.
                     format(ident['type'], ident['name']))
            discovery = manager.process_identity(user, contact,
                                                 ident['name'], ident['type'])
            if discovery:
                founds.append(discovery)
        if founds:
            log.info('Found %d results' % len(founds))
            self._process_results(user, contact, founds)

def
process_key_discovery
(
self
,
msg
,
payload
)
:
if
'user_id'
not
in
payload
or
'contact_id'
not
in
payload
:
raise
Exception
(
'Invalid contact_update structure'
)
user
=
User
.
get
(
payload
[
'user_id'
]
)
contact
=
Contact
(
user
.
user_id
,
contact_id
=
payload
[
'contact_id'
]
)
contact
.
get_db
(
)
contact
.
unmarshall_db
(
)
manager
=
ContactPublicKeyManager
(
)
founds
=
[
]
for
ident
in
payload
.
get
(
'emails'
,
[
]
)
:
log
.
info
(
'Process email identity {0}'
.
format
(
ident
[
'address'
]
)
)
discovery
=
manager
.
process_identity
(
user
,
contact
,
ident
[
'address'
]
,
'email'
)
if
discovery
:
founds
.
append
(
discovery
)
for
ident
in
payload
.
get
(
'identities'
,
[
]
)
:
log
.
info
(
'Process identity {0}:{1}'
.
format
(
ident
[
'type'
]
,
ident
[
'name'
]
)
)
discovery
=
manager
.
process_identity
(
user
,
contact
,
ident
[
'name'
]
,
ident
[
'type'
]
)
if
discovery
:
founds
.
append
(
discovery
)
if
founds
:
log
.
info
(
'Found %d results'
%
len
(
founds
)
)
self
.
_process_results
(
user
,
contact
,
founds
)


def handler(self, msg):
        """Handle a discover_key nats messages."""
        payload = json.loads(msg.data)
        if payload['order'] == "discover_key":
            self.process_key_discovery(msg, payload)
        else:
            log.warn(
                'Unhandled payload order "{}" \
                (queue : keyQueue, subject : keyAction)'.format(
                    payload['order']))
            raise NotImplementedError


Create a new UserMessageDelivery belong to an user .

def __init__(self, user, identity):
        """Create a new UserMessageDelivery belong to an user."""
        self.user = user
        self.identity = identity
        self.qualifier = None

def __init__ ( self , user , identity ) : self . user = user self . identity = identity self . qualifier = None


Process a raw message for an user ie makes it a rich message .

def process_raw(self, raw_msg_id):
        """Process a raw message for an user, ie makes it a rich 'message'."""
        raw = RawMessage.get(raw_msg_id)
        if not raw:
            log.error('Raw message <{}> not found'.format(raw_msg_id))
            raise NotFound
        log.debug('Retrieved raw message {}'.format(raw_msg_id))

        message = self.qualifier.process_inbound(raw)
        # fill user_tag table with imported tags and embed in message
        for tag in message.ext_tags:
            try:
                Tag.get(user_id=self.user.user_id, name=tag.name)
            except NotFound:
                try:
                    Tag.create(user_id=self.user.user_id, name=tag.name,
                               label=tag.label, type=tag.type,
                               date_insert=datetime.datetime.now(tz=pytz.utc))
                except Exception as exc:
                    log.exception(
                        "UserMessageDelivery failed to create tag : {}".format(
                            exc))
            message.tags.append(tag.name)

        if message.external_msg_id:
            external_refs = Merl._model_class.filter(
                user_id=self.user.user_id,
                external_msg_id=message.external_msg_id)
            if external_refs:
                msg = external_refs[0]
                # message already imported, should update it with identity_id ?
                obj = Message(user=self.user,
                              message_id=msg.message_id)
                if str(msg.identity_id) != self.identity.identity_id:
                    obj.get_db()
                    obj.unmarshall_db()
                    obj.user_identities.append(self.identity.identity_id)
                    obj.marshall_db()
                    obj.save_db()
                    obj.marshall_index()
                    obj.save_index()
                    Merl.create(self.user,
                                external_msg_id=msg.external_msg_id,
                                identity_id=self.identity.identity_id,
                                message_id=msg.message_id)
                    # TODO: update flags ?
                raise DuplicateObject(DUPLICATE_MESSAGE_EXC)
        else:
            log.warn('Message without external message_id for raw {}'.
                     format(raw.raw_msg_id))
        # store and index Message
        obj = Message(user=self.user)
        obj.unmarshall_dict(message.to_native())
        obj.user_id = uuid.UUID(self.user.user_id)
        obj.user_identities = [uuid.UUID(self.identity.identity_id)]
        obj.message_id = uuid.uuid4()
        obj.date_insert = datetime.datetime.now(tz=pytz.utc)
        obj.date_sort = obj.date_insert
        obj.marshall_db()
        obj.save_db()
        obj.marshall_index()
        obj.save_index()

        if message.external_msg_id:
            # store external_msg_id in lookup table
            # but do not abort if it failed
            try:
                Merl.create(self.user,
                            external_msg_id=obj.external_msg_id,
                            identity_id=obj.user_identity,
                            message_id=obj.message_id)
            except Exception as exc:
                log.exception("UserMessageDelivery failed "
                              "to store message_external_ref : {}".format(exc))

        return obj


def inbound_smtp_handler(config):
    """Inbound message NATS handler."""
    client = Nats()
    server = 'nats://{}:{}'.format(config['host'], config['port'])
    servers = [server]

    opts = {"servers": servers}
    yield client.connect(**opts)

    # create and register subscriber(s)
    inbound_email_sub = subscribers.InboundEmail(client)
    future = client.subscribe("inboundSMTP", "SMTPqueue",
                              inbound_email_sub.handler)
    log.info("nats subscription started for inboundSMTP")
    future.result()


def inbound_twitter_handler(config):
    """Inbound twitterDM NATS handler"""
    client = Nats()
    server = 'nats://{}:{}'.format(config['host'], config['port'])
    servers = [server]

    opts = {"servers": servers}
    yield client.connect(**opts)

    # create and register subscriber(s)
    inbound_twitter_sub = subscribers.InboundTwitter(client)
    future = client.subscribe("inboundTwitter", "Twitterqueue",
                              inbound_twitter_sub.handler)
    log.info("nats subscription started for inboundTwitter")
    future.result()


def inbound_mastodon_handler(config):
    """Inbound mastodonDM NATS handler"""
    client = Nats()
    server = 'nats://{}:{}'.format(config['host'], config['port'])
    servers = [server]

    opts = {"servers": servers}
    yield client.connect(**opts)

    # create and register subscriber(s)
    inbound_mastodon_sub = subscribers.InboundMastodon(client)
    future = client.subscribe("inboundMastodon", "Mastodonqueue",
                              inbound_mastodon_sub.handler)
    log.info("nats subscription started for inboundMastodon")
    future.result()


NATS handler for contact update events .

def contact_handler(config):
    """NATS handler for contact update events."""
    client = Nats()
    server = 'nats://{}:{}'.format(config['host'], config['port'])
    servers = [server]

    opts = {"servers": servers}
    yield client.connect(**opts)

    # create and register subscriber(s)
    contact_subscriber = subscribers.ContactAction(client)
    future = client.subscribe("contactAction", "contactQueue",
                     contact_subscriber.handler)
    log.info("nats subscription started for contactAction")
    future.result()


def key_handler(config):
    """NATS handler for discover_key events."""
    client = Nats()
    server = 'nats://{}:{}'.format(config['host'], config['port'])
    servers = [server]

    opts = {"servers": servers}
    yield client.connect(**opts)

    # create and register subscriber(s)
    key_subscriber = subscribers.KeyAction(client)
    future = client.subscribe("keyAction",
                              "keyQueue",
                              key_subscriber.handler)
    log.info("nats subscription started for keyAction")
    future.result()


Caliopen entry point for WSGI application .

def main(global_config, **settings):
    """Caliopen entry point for WSGI application.

    Load Caliopen configuration and setup a WSGI application
    with loaded API services.
    """
    # XXX ugly way to init caliopen configuration before pyramid
    caliopen_config = settings['caliopen.config'].split(':')[1]
    Configuration.load(caliopen_config, 'global')

    settings['pyramid_swagger.exclude_paths'] = [r'^/api-ui/?', r'^/doc/api/?', r'^/defs/?']
    settings['pyramid_swagger.enable_response_validation'] = True
    config = Configurator(settings=settings)
    services = config.registry.settings. \
        get('caliopen_api.services', []). \
        split('\n')
    route_prefix = settings.get('caliopen_api.route_prefix')
    for service in services:
        log.info('Loading %s service' % service)
        config.include(service, route_prefix=route_prefix)
    config.end()
    return config.make_wsgi_app()


Return pagination limit from request else default .

def get_limit(self):
        """Return pagination limit from request else default."""
        return int(self.request.params.get('limit', DEFAULT_LIMIT))

def get_limit ( self ) : return int ( self . request . params . get ( 'limit' , DEFAULT_LIMIT ) )


Return pagination offset from request else 0 .

def get_offset(self):
        """Return pagination offset from request else 0."""
        return int(self.request.params.get('offset', 0))

def get_offset ( self ) : return int ( self . request . params . get ( 'offset' , 0 ) )


Format error details for a server error .

def format_response_detail(exc, request):
    """Format error details for a server error."""
    route = (request.matched_route.name
             if request.matched_route else ('Unknown'))
    return {'component': route,
            'values': exc.code,
            'property': exc.__class__.__name__,
            'message': exc.message}

def format_response_detail ( exc , request ) : route = ( request . matched_route . name if request . matched_route else ( 'Unknown' ) ) return { 'component' : route , 'values' : exc . code , 'property' : exc . __class__ . __name__ , 'message' : exc . message }


Format response error details contains application errors .

def format_response(exc, request, details=None):
    """Format response error, details contains application errors."""
    # XXX better design for details as keys are not validated
    if not details:
        details = {'values': [exc.code],
                   'property': None,
                   'component': 'server'}
    error = {"errors": [{"description": exc.explanation,
                         "type": exc.title,
                         "values": details['values'],
                         "property": details['property'],
                         "component": details['component'],
                         "message": "{}".format(details['message']),
                         "code": exc.code}]}
    response = Response(json.dumps(error))
    response.content_type = str('application/json; charset=UTF-8')
    response.status_int = exc.code
    return response


def http_forbidden(exc, request):
    """Raise HTTPUnauthorized exception."""
    if request.authenticated_userid is None:
        exc = HTTPUnauthorized(explanation="Invalid credentials")
    return http_exception(exc, request)

def
http_forbidden
(
exc
,
request
)
:
if
request
.
authenticated_userid
is
:
exc
=
HTTPUnauthorized
(
explanation
=
"Invalid credentials"
)
return
http_exception
(
exc
,
request
)


def default(self, obj):
        '''Convert object to JSON encodable type.'''
        if isinstance(obj, Decimal):
            return float(obj)
        if isinstance(obj, datetime.datetime):
            if obj.tzinfo is None:
                return str(obj.replace(tzinfo=pytz.utc))
        if isinstance(obj, datetime.date):
            return obj.isoformat()
        if isinstance(obj, UUID):
            return str(obj)
        return super(JSONEncoder, self).default(obj)

def default ( self , obj ) : if isinstance ( obj , Decimal ) : return float ( obj ) if isinstance ( obj , datetime . datetime ) : if obj . tzinfo is None : return str ( obj . replace ( tzinfo = pytz . utc ) ) if isinstance ( obj , datetime . date ) : return obj . isoformat ( ) if isinstance ( obj , UUID ) : return str ( obj ) return super ( JSONEncoder , self ) . default ( obj )


def json_deserializer(request):
    """Manage json content type."""
    if request.json_body:
        return request.json_body
    return request.body

def json_deserializer ( request ) : if request . json_body : return request . json_body return request . body


def authenticated_user(self):
        """
        Return the authenticated user.

        :return: authenticated user
        :rtype: dict
        """
        user = self.request.authenticated_userid
        return user

def authenticated_user ( self ) : user = self . request . authenticated_userid return user


def swagger_error_view(exc, request):
    """Format swagger validation error."""
    raise ValidationError(exc.child.message)

def swagger_error_view ( exc , request ) : raise ValidationError ( exc . child . message )


def includeme(config):
    """Configure REST API."""
    connect_storage()
    config.commit()

    # configure renderers
    config.add_renderer('text_plain', TextPlainRenderer)
    config.add_renderer('json', JsonRenderer)
    config.add_renderer('simplejson', JsonRenderer)
    config.add_renderer('part', PartRenderer)

    # configure specific views for API errors
    config.scan('caliopen_api.base.errors')
    config.add_cornice_deserializer('application/json',
                                    json_deserializer)
    swagger_context = 'pyramid_swagger.exceptions.RequestValidationError'
    config.add_exception_view(context=swagger_context,
                              view=swagger_error_view,
                              renderer='json')
    config.commit()
    log.info('Base API configured')


def _validate_signature(self, request, device_id, infos):
        """Validate device signature."""
        if infos['curve'] == 'P-256':
            curve = ecdsa.ecdsa.curve_256
            crv = ecdsa.curves.NIST256p
            hashfunc = hashlib.sha256
        else:
            log.warn('Unsupported curve %r' % infos['curve'])
            return False
        try:
            point = ecdsa.ellipticcurve.Point(curve, infos['x'], infos['y'])
        except AssertionError:
            log.warn('Invalid curve points')
            return False

        sign_header = request.headers.get('X-Caliopen-Device-Signature', None)
        if sign_header:
            data = '{}{}{}'.format(request.method, request.path_qs, '')
            try:
                b64_header = base64.decodestring(sign_header)
                ecdsasign = EcdsaSignature.load(b64_header)
                signature = ecdsasign['r'].contents + ecdsasign['s'].contents
                vk = ecdsa.VerifyingKey.from_public_point(point, crv,
                                                          hashfunc=hashfunc)
                return vk.verify(signature, data)
            except ecdsa.BadSignatureError:
                pass
            except Exception as exc:
                log.error('Exception during signature verification %r' % exc)
            return False
        else:
            log.warn('no device signature')
            return False


Token Key mechanism can t remember anyone .

def remember(self, request, principal, **kw):
        """Token Key mechanism can't remember anyone."""
        return []

def remember ( self , request , principal , * * kw ) : return [ ]


def get_device_sig_key(user, device):
    """Get device signature key."""
    keys = PublicKey._model_class.filter(user_id=user.user_id,
                                         resource_id=device.device_id)
    keys = [x for x in keys if x.resource_type == 'device' and
            x.use == 'sig']
    if keys:
        return keys[0]
    return None

def get_device_sig_key ( user , device ) : keys = PublicKey . _model_class . filter ( user_id = user . user_id , resource_id = device . device_id ) keys = [ x for x in keys if x . resource_type == 'device' and x . use == 'sig' ] if keys : return keys [ 0 ] return None


Patch a device signature public key as X and Y points are not valid .

def patch_device_key(key, param):
    """Patch a device signature public key as X and Y points are not valid."""
    if not key.x and not key.y:
        key.x = int(param['ecdsa_key']['x'], 16)
        key.y = int(param['ecdsa_key']['y'], 16)
        key.save()
        return True
    return False

def patch_device_key ( key , param ) : if not key . x and not key . y : key . x = int ( param [ 'ecdsa_key' ] [ 'x' ] , 16 ) key . y = int ( param [ 'ecdsa_key' ] [ 'y' ] , 16 ) key . save ( ) return True return False


Return ( key tokens ) informations for cache entry management .

def make_user_device_tokens(request, user, device, key, ttl=86400):
    """Return (key, tokens) informations for cache entry management."""
    cache_key = '{}-{}'.format(user.user_id, device.device_id)

    previous = request.cache.get(cache_key)
    if previous:
        status = previous.get('user_status', 'unknown')
        log.info('Found current user device entry {} : {}'.
                 format(cache_key, status))
        if status in ['locked', 'maintenance']:
            raise AuthenticationError('Status {} does not permit operations'.
                                      format(status))

    access_token = create_token()
    refresh_token = create_token(80)

    expires_at = (datetime.datetime.utcnow() +
                  datetime.timedelta(seconds=ttl))

    tokens = {'access_token': access_token,
              'refresh_token': refresh_token,
              'expires_in': ttl,  # TODO : remove this value
              'shard_id': user.shard_id,
              'expires_at': expires_at.isoformat(),
              'user_status': 'active',
              'key_id': str(key.key_id),
              'x': key.x,
              'y': key.y,
              'curve': key.crv}

    request.cache.set(cache_key, tokens)
    result = tokens.copy()
    result.pop('shard_id')
    return result


def no_such_user(request):
    """Validator that an user does not exist."""
    username = request.swagger_data['user']['username']
    if not User.is_username_available(username):
        raise NotAcceptable(detail='User already exist')

def
no_such_user
(
request
)
:
username
=
request
.
swagger_data
[
'user'
]
[
'username'
]
if
not
User
.
is_username_available
(
username
)
:
raise
NotAcceptable
(
detail
=
'User already exist'
)


Configure REST API for user and contact .

def includeme(config):
    """Configure REST API for user and contact."""
    config.set_authentication_policy(AuthenticationPolicy())
    config.set_authorization_policy(AuthorizationPolicy())
    log.debug('Loading user API')
    config.scan('caliopen_api.user.user')

    log.debug('Loading contact API')
    config.scan('caliopen_api.user.contact')

    log.debug('Loading imports API')
    config.scan('caliopen_api.user.imports')

    log.debug('Loading settings API')
    config.scan('caliopen_api.user.settings')

def
includeme
(
config
)
:
config
.
set_authentication_policy
(
AuthenticationPolicy
(
)
)
config
.
set_authorization_policy
(
AuthorizationPolicy
(
)
)
log
.
debug
(
'Loading user API'
)
config
.
scan
(
'caliopen_api.user.user'
)
log
.
debug
(
'Loading contact API'
)
config
.
scan
(
'caliopen_api.user.contact'
)
log
.
debug
(
'Loading imports API'
)
config
.
scan
(
'caliopen_api.user.imports'
)
log
.
debug
(
'Loading settings API'
)
config
.
scan
(
'caliopen_api.user.settings'
)


def includeme(config):
    """
    Serve discussion related REST API.
    """

    config.commit()

    log.debug('Loading participants discussion API')
    config.scan('caliopen_api.discussion.participants')

def
includeme
(
config
)
:
config
.
commit
(
)
log
.
debug
(
'Loading participants discussion API'
)
config
.
scan
(
'caliopen_api.discussion.participants'
)


def includeme(config):
    """
    Serve message and discussion REST API.
    """

    config.commit()

    # Activate cornice in any case and scan
    log.debug('Loading message API')
    config.scan('caliopen_api.message.message')

def
includeme
(
config
)
:
config
.
commit
(
)
# Activate cornice in any case and scan
log
.
debug
(
'Loading message API'
)
config
.
scan
(
'caliopen_api.message.message'
)


def create_new_index(self, index):
        """Create user index and setups mappings."""
        log.info('Creating new index {}'.format(index))

        try:
            self.es_client.indices.create(
                index=index,
                timeout='30s',
                body={
                    "settings": {
                        "index": {
                            "number_of_shards": 3,
                        },
                        "analysis": {
                            "analyzer": {
                                "text_analyzer": {
                                    "type": "custom",
                                    "tokenizer": "lowercase",
                                    "filter": [
                                        "ascii_folding"
                                    ]
                                },
                                "email_analyzer": {
                                    "type": "custom",
                                    "tokenizer": "email_tokenizer",
                                    "filter": [
                                        "ascii_folding"
                                    ]
                                }
                            },
                            "filter": {
                                "ascii_folding": {
                                    "type": "asciifolding",
                                    "preserve_original": True
                                }
                            },
                            "tokenizer": {
                                "email_tokenizer": {
                                    "type": "ngram",
                                    "min_gram": 3,
                                    "max_gram": 25
                                }
                            }
                        }
                    }
                })
        except Exception as exc:
            log.error("failed to create index {} : {}".format(index, exc))
            raise exc

        # PUT mappings for each type
        for kls in self.types:
            kls._index_class().create_mapping(index)

def
create_new_index
(
self
,
index
)
:
log
.
info
(
'Creating new index {}'
.
format
(
index
)
)
try
:
self
.
es_client
.
indices
.
create
(
index
=
index
,
timeout
=
'30s'
,
body
=
{
"settings"
:
{
"index"
:
{
"number_of_shards"
:
3
,
}
,
"analysis"
:
{
"analyzer"
:
{
"text_analyzer"
:
{
"type"
:
"custom"
,
"tokenizer"
:
"lowercase"
,
"filter"
:
[
"ascii_folding"
]
}
,
"email_analyzer"
:
{
"type"
:
"custom"
,
"tokenizer"
:
"email_tokenizer"
,
"filter"
:
[
"ascii_folding"
]
}
}
,
"filter"
:
{
"ascii_folding"
:
{
"type"
:
"asciifolding"
,
"preserve_original"
:
True
}
}
,
"tokenizer"
:
{
"email_tokenizer"
:
{
"type"
:
"ngram"
,
"min_gram"
:
3
,
"max_gram"
:
25
}
}
}
}
}
)
except
Exception
as
exc
:
log
.
error
(
"failed to create index {} : {}".
f
ormat(
i
ndex,
xc)
)
raise
exc
# PUT mappings for each type
for
kls
in
self
.
types
:
kls
.
_index_class
(
)
.
create_mapping
(
index
)


fill_date_sort updates all message docs in index with either message s date or date_insert into date_sort depending of message being received or sent

def fill_date_sort(self, index):
        """
        fill_date_sort updates all message docs in index
        with either message's date or date_insert into date_sort,
        depending of message being received or sent

        :param index: Elasticsearch index
        :return: None
        """
        log.info("filling date_sort prop for index {}".format(index))
        q = {
            "script": {
                "lang": "painless",
                "inline": "if (ctx._source.is_received) {ctx._source.date_sort = ctx._source.date_insert} else {ctx._source.date_sort = ctx._source.date}"
            }
        }
        try:
            resp = requests.post(self.url + "/" + index + "/indexed_message/_update_by_query", data=json.dumps(q), headers={'content-type': 'application/json'})
            log.info("{} docs updated in {}".format(resp.json()["updated"], index))
        except Exception as exc:
            log.error(
                "failed to update_by_query index {} : {}".format(index, exc))


Creates user index and setups mappings .

def create_new_index(self, index):
        """Creates user index and setups mappings."""
        log.warn('Creating new index {}'.format(index))

        try:
            self.es_client.indices.create(
                index=index,
                body={
                    "settings": {
                        "analysis": {
                            "analyzer": {
                                "text_analyzer": {
                                    "type": "custom",
                                    "tokenizer": "lowercase",
                                    "filter": [
                                        "ascii_folding"
                                    ]
                                },
                                "email_analyzer": {
                                    "type": "custom",
                                    "tokenizer": "email_tokenizer",
                                    "filter": [
                                        "ascii_folding"
                                    ]
                                }
                            },
                            "filter": {
                                "ascii_folding": {
                                    "type": "asciifolding",
                                    "preserve_original": True
                                }
                            },
                            "tokenizer": {
                                "email_tokenizer": {
                                    "type": "ngram",
                                    "min_gram": 3,
                                    "max_gram": 25
                                }
                            }
                        }
                    }
                })
        except Exception as exc:
            log.error("failed to create index {} : {}".format(index, exc))
            raise exc

        # PUT mappings for each type
        for kls in self.types:
            kls._index_class().create_mapping(index)

def
create_new_index
(
self
,
index
)
:
log
.
warn
(
'Creating new index {}'
.
format
(
index
)
)
try
:
self
.
es_client
.
indices
.
create
(
index
=
index
,
body
=
{
"settings"
:
{
"analysis"
:
{
"analyzer"
:
{
"text_analyzer"
:
{
"type"
:
"custom"
,
"tokenizer"
:
"lowercase"
,
"filter"
:
[
"ascii_folding"
]
}
,
"email_analyzer"
:
{
"type"
:
"custom"
,
"tokenizer"
:
"email_tokenizer"
,
"filter"
:
[
"ascii_folding"
]
}
}
,
"filter"
:
{
"ascii_folding"
:
{
"type"
:
"asciifolding"
,
"preserve_original"
:
True
}
}
,
"tokenizer"
:
{
"email_tokenizer"
:
{
"type"
:
"ngram"
,
"min_gram"
:
3
,
"max_gram"
:
25
}
}
}
}
}
)
except
Exception
as
exc
:
log
.
error
(
"failed to create index {} : {}".
f
ormat(
i
ndex,
xc)
)
raise
exc
# PUT mappings for each type
for
kls
in
self
.
types
:
kls
.
_index_class
(
)
.
create_mapping
(
index
)