Client transp fix.

git-svn-id: https://svn.calendarserver.org/repository/calendarserver/CalendarServer/branches/release/CalendarServer-5.3-dev@14169 e27351fd-9f3e-4f54-a53b-843176b1656c

support/build.sh

@@ -809,7 +809,7 @@ dependencies () {
 
   # XXX actually PyCalendar should be imported in-place.
   py_dependency -fe -i "src" -r HEAD \
-    "PyCalendar" "pycalendar" "pycalendar" \
+    "PyCalendar" "pycalendar" "pycalendar-2" \
     "${svn_uri_base}/PyCalendar/branches/CalendarServer-5.2";
 
   #

twext/python/launchd.py

@@ -40,9 +40,9 @@
 
 ffi.cdef("""
 
-static const char* LAUNCH_KEY_CHECKIN;
-static const char* LAUNCH_JOBKEY_LABEL;
-static const char* LAUNCH_JOBKEY_SOCKETS;
+static char const* const LAUNCH_KEY_CHECKIN;
+static char const* const LAUNCH_JOBKEY_LABEL;
+static char const* const LAUNCH_JOBKEY_SOCKETS;
 
 typedef enum {
     LAUNCH_DATA_DICTIONARY = 1,

twistedcaldav/stdconfig.py

@@ -866,6 +866,16 @@
     # How many results to return for principal-property-search REPORT requests
     "MaxPrincipalSearchReportResults": 500,
 
+    #
+    # Client fixes per user-agent match
+    #
+    "ClientFixes" : {
+        "ForceAttendeeTRANSP" : [
+            "iOS/8\\.0(\\..*)?",
+            "iOS/8\\.1(\\..*)?",
+        ],
+    },
+
     #
     # Localization
     #
@@ -1427,6 +1437,19 @@ def _updateRejectClients(configDict, reloading=False):
 
 
 
+def _updateClientFixes(configDict, reloading=False):
+    #
+    # Compile ClientFixes expressions for speed
+    #
+    try:
+        configDict.ClientFixesCompiled = {}
+        for key, expressions in configDict.ClientFixes.items():
+            configDict.ClientFixesCompiled[key] = [re.compile("^{}$".format(x)) for x in expressions]
+    except re.error, e:
+        raise ConfigurationError("Invalid regular expression in ClientFixes: %s" % (e,))
+
+
+
 def _updateLogLevels(configDict, reloading=False):
     log.publisher.levels.clearLogLevels()
 
@@ -1625,6 +1648,7 @@ def _updateCompliance(configDict, reloading=False):
     _postUpdateProxyDBService,
     _updateACLs,
     _updateRejectClients,
+    _updateClientFixes,
     _updateLogLevels,
     _updateNotifications,
     _updateScheduling,

twistedcaldav/storebridge.py

@@ -55,6 +55,7 @@
 from twistedcaldav.resource import CalDAVResource, GlobalAddressBookResource, \
     DefaultAlarmPropertyMixin
 from twistedcaldav.scheduling_store.caldav.resource import ScheduleInboxResource
+from twistedcaldav.util import matchClientFixes
 from twistedcaldav.vcard import Component as VCard, InvalidVCardDataError
 
 from txdav.base.propertystore.base import PropertyName
@@ -2809,6 +2810,10 @@ def http_PUT(self, request):
                     "Can't parse calendar data"
                 ))
 
+            # Look for client fixes
+            ua = request.headers.getHeader("User-Agent")
+            self._newStoreParent._txn._client_fixes = matchClientFixes(config, ua)
+
             # storeComponent needs to know who the auth'd user is for access control
             # TODO: this needs to be done in a better way - ideally when the txn is created for the request,
             # we should set a txn.authzid attribute.

twistedcaldav/test/test_util.py

@@ -0,0 +1,98 @@
+##
+# Copyright (c) 2005-2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+import twistedcaldav.test.util
+from twistedcaldav.util import userAgentProductTokens, matchClientFixes
+from twistedcaldav.stdconfig import _updateClientFixes
+from twistedcaldav.config import ConfigDict
+
+class TestUtil(twistedcaldav.test.util.TestCase):
+    """
+    Tests for L{twistedcaldav.util}
+    """
+    def test_userAgentProductTokens(self):
+        """
+        Test that L{userAgentProductTokens} correctly parses a User-Agent header.
+        """
+        for hdr, result in (
+            # Valid syntax
+            ("Client/1.0", ["Client/1.0", ]),
+            ("Client/1.0 FooBar/2", ["Client/1.0", "FooBar/2", ]),
+            ("Client/1.0 (commentary here)", ["Client/1.0", ]),
+            ("Client/1.0 (FooBar/2)", ["Client/1.0", ]),
+            ("Client/1.0 (commentary here) FooBar/2", ["Client/1.0", "FooBar/2", ]),
+            ("Client/1.0 (commentary here) FooBar/2 (more commentary here) ", ["Client/1.0", "FooBar/2", ]),
+
+            # Invalid syntax
+            ("Client/1.0 (commentary here FooBar/2", ["Client/1.0", ]),
+            ("Client/1.0 commentary here) FooBar/2", ["Client/1.0", "commentary", "here)", "FooBar/2", ]),
+        ):
+            self.assertEqual(userAgentProductTokens(hdr), result, msg="Mismatch: {}".format(hdr))
+
+
+    def test_matchClientFixes(self):
+        """
+        Test that L{matchClientFixes} correctly identifies clients with matching fix tokens.
+        """
+        c = ConfigDict()
+        c.ClientFixes = {
+            "fix1": [
+                "Client/1\\.0.*",
+                "Client/1\\.1(\\..*)?",
+                "Client/2",
+            ],
+            "fix2": [
+                "Other/1\\.0.*",
+            ],
+        }
+        _updateClientFixes(c)
+        _updateClientFixes(c)
+
+        # Valid matches
+        for ua in (
+            "Client/1.0 FooBar/2",
+            "Client/1.0.1 FooBar/2",
+            "Client/1.0.1.1 FooBar/2",
+            "Client/1.1 FooBar/2",
+            "Client/1.1.1 FooBar/2",
+            "Client/2 FooBar/2",
+        ):
+            self.assertEqual(
+                matchClientFixes(c, ua),
+                set(("fix1",)),
+                msg="Did not match {}".format(ua),
+            )
+
+        # Valid non-matches
+        for ua in (
+            "Client/1 FooBar/2",
+            "Client/1.10 FooBar/2",
+            "Client/2.0 FooBar/2",
+            "Client/2.0.1 FooBar/2",
+            "Client FooBar/2",
+            "Client/3 FooBar/2",
+            "Client/3.0 FooBar/2",
+            "Client/10 FooBar/2",
+            "Client/10.0 FooBar/2",
+            "Client/10.0.1 FooBar/2",
+            "Client/10.0.1 (Client/1.0) FooBar/2",
+            "Client/10.0.1 (foo Client/1.0 bar) FooBar/2",
+        ):
+            self.assertEqual(
+                matchClientFixes(c, ua),
+                set(),
+                msg="Incorrectly matched {}".format(ua),
+            )

twistedcaldav/util.py

@@ -18,6 +18,7 @@
 import re
 import sys
 import base64
+import itertools
 
 from subprocess import Popen, PIPE, STDOUT
 from hashlib import md5, sha1
@@ -517,3 +518,57 @@ def normalizationLookup(cuaddr, principalFunction, config):
         cuas = principal.record.calendarUserAddresses
 
         return (fullName, rec.guid, cuas)
+
+
+
+def userAgentProductTokens(user_agent):
+    """
+    Parse an HTTP User-Agent header to extract the product tokens and ignore
+    any parenthesized comment strings in the header.
+
+    @param user_agent: text of User-Agent header value
+    @type user_agent: L{str}
+
+    @return: list of product tokens extracted from the header
+    @rtype: L{list}
+    """
+
+    ua_hdr = user_agent.split()
+    ua_tokens = []
+    comment = False
+    for token in ua_hdr:
+        if comment:
+            if token.endswith(")"):
+                comment = False
+        elif token.startswith("("):
+            if not token.endswith(")"):
+                comment = True
+        else:
+            ua_tokens.append(token)
+
+    return ua_tokens
+
+
+
+def matchClientFixes(config, user_agent):
+    """
+    Given a user-agent string, see if it matches any of the configured client fixes.
+
+    @param config: the L{config} to match against.
+    @type config: L{ConfigDict}
+    @param user_agent: the HTTP User-Agent header value to test.
+    @type user_agent: L{str}
+    """
+
+    if len(config.ClientFixesCompiled) == 0 or not user_agent:
+        return set()
+
+    ua_tokens = userAgentProductTokens(user_agent)
+
+    client_fixes = set()
+    for fix, patterns in config.ClientFixesCompiled.items():
+        for pattern, token in itertools.product(patterns, ua_tokens):
+            if pattern.match(token) is not None:
+                client_fixes.add(fix)
+                break
+    return client_fixes

txdav/caldav/datastore/scheduling/icaldiff.py

@@ -27,30 +27,40 @@
 from txdav.caldav.datastore.scheduling.cuaddress import normalizeCUAddr
 from txdav.caldav.datastore.scheduling.itip import iTipGenerator
 
-"""
-Class that handles diff'ing two calendar objects.
-"""
-
 __all__ = [
     "iCalDiff",
 ]
 
 log = Logger()
 
 class iCalDiff(object):
+    """
+    This object is used for doing comparisons between two calendar objects to
+    work out what the changes are, in order to determine whether a scheduling
+    operation might be needed. The behavior will varying based on whether the
+    change is being triggered by an Organizer or an Attendee.
+    """
 
-    def __init__(self, oldcalendar, newcalendar, smart_merge):
-        """
 
-        @param oldcalendar:
-        @type oldcalendar:
-        @param newcalendar:
-        @type newcalendar:
+    def __init__(self, oldcalendar, newcalendar, smart_merge, forceTRANSP=False):
+        """
+        Note that this object will always duplicate the calendar objects when doing
+        comparisons so as not to change the calendar objects passed in.
+
+        @param oldcalendar: the existing calendar object to compare to
+        @type oldcalendar: L{Component}
+        @param newcalendar: the new calendar object
+        @type newcalendar: L{Component}
+        @param smart_merge: whether or not a "smart" CalDAV merge is done (If-Schedule-Tag-Match)
+        @type smart_merge: L{bool}
+        @param forceTRANSP: whether or not to apply a fix for clients failing to set TRANSP properly
+        @type forceTRANSP: L{bool}
         """
 
         self.oldcalendar = oldcalendar
         self.newcalendar = newcalendar
         self.smart_merge = smart_merge
+        self.forceTRANSP = forceTRANSP
 
 
     def organizerDiff(self):
@@ -501,6 +511,13 @@ def _transferAttendeeData(self, serverComponent, clientComponent, declines):
 
             replyNeeded = True
 
+            # Apply client fix only if the PARTSTAT was changed
+            if self.forceTRANSP:
+                if clientAttendee.parameterValue("PARTSTAT", "NEEDS-ACTION") in ("ACCEPTED", "TENTATIVE",):
+                    clientComponent.replaceProperty(Property("TRANSP", "OPAQUE"))
+                else:
+                    clientComponent.replaceProperty(Property("TRANSP", "TRANSPARENT"))
+
         if serverAttendee.parameterValue("RSVP", "FALSE") != clientAttendee.parameterValue("RSVP", "FALSE"):
             if clientAttendee.parameterValue("RSVP", "FALSE") == "FALSE":
                 try:

txdav/caldav/datastore/scheduling/implicit.py

@@ -1355,7 +1355,14 @@ def isAttendeeChangeInsignificant(self):
                     (caldav_namespace, "valid-attendee-change"),
                     "Cannot use an event when not listed as an attendee in the organizer's copy",
                 ))
-        differ = iCalDiff(oldcalendar, self.calendar, self.do_smart_merge)
+
+        # Check for a required client fix
+        if hasattr(self.txn, "_client_fixes"):
+            forceTRANSP = "ForceAttendeeTRANSP" in self.txn._client_fixes
+        else:
+            forceTRANSP = False
+
+        differ = iCalDiff(oldcalendar, self.calendar, self.do_smart_merge, forceTRANSP=forceTRANSP)
         return differ.attendeeMerge(self.attendee)