def _get_ideas_real(discussion, view_def=None, ids=None, user_id=None):
    user_id = user_id or Everyone
    # optimization: Recursive widget links.
    from assembl.models import (
        Widget, IdeaWidgetLink, IdeaDescendantsShowingWidgetLink)
    universal_widget_links = []
    by_idea_widget_links = defaultdict(list)
    widget_links = discussion.db.query(IdeaWidgetLink
        ).join(Widget).join(Discussion).filter(
        Widget.test_active(), Discussion.id == discussion.id,
        IdeaDescendantsShowingWidgetLink.polymorphic_filter()
        ).options(joinedload_all(IdeaWidgetLink.idea)).all()
    for wlink in widget_links:
        if isinstance(wlink.idea, RootIdea):
            universal_widget_links.append({
                '@type': wlink.external_typename(),
                'widget': Widget.uri_generic(wlink.widget_id)})
        else:
            for id in wlink.idea.get_all_descendants(True):
                by_idea_widget_links[Idea.uri_generic(id)].append({
                    '@type': wlink.external_typename(),
                    'widget': Widget.uri_generic(wlink.widget_id)})

    next_synthesis = discussion.get_next_synthesis()
    ideas = discussion.db.query(Idea).filter_by(
        discussion_id=discussion.id
    )

    ideas = ideas.outerjoin(SubGraphIdeaAssociation,
                    and_(SubGraphIdeaAssociation.sub_graph_id==next_synthesis.id, SubGraphIdeaAssociation.idea_id==Idea.id)
        )
    
    ideas = ideas.outerjoin(IdeaLink,
                    and_(IdeaLink.target_id==Idea.id)
        )
    
    ideas = ideas.order_by(IdeaLink.order, Idea.creation_date)
    
    if ids:
        ids = [get_database_id("Idea", id) for id in ids]
        ideas = ideas.filter(Idea.id.in_(ids))
    # remove tombstones
    ideas = ideas.filter(and_(*Idea.base_conditions()))
    ideas = ideas.options(
        joinedload_all(Idea.source_links),
        joinedload_all(Idea.has_showing_widget_links),
        undefer(Idea.num_children))

    permissions = get_permissions(user_id, discussion.id)
    Idea.prepare_counters(discussion.id, True)
    retval = [idea.generic_json(view_def, user_id, permissions)
              for idea in ideas]
    retval = [x for x in retval if x is not None]
    for r in retval:
        if r.get('widget_links', None) is not None:
            links = r['widget_links'][:]
            links.extend(universal_widget_links)
            links.extend(by_idea_widget_links[r['@id']])
            r['active_widget_links'] = links
    return retval

def create_idea(request):
    discussion_id = int(request.matchdict['discussion_id'])
    session = Discussion.default_db
    discussion = session.query(Discussion).get(int(discussion_id))
    user_id = request.authenticated_userid
    permissions = get_permissions(user_id, discussion.id)
    idea_data = json.loads(request.body)
    kwargs = {
        "discussion": discussion
    }

    for key, attr_name in langstring_fields.iteritems():
        if key in idea_data:
            ls_data = idea_data[key]
            if ls_data is None:
                continue
            assert isinstance(ls_data, dict)
            current = LangString.create_from_json(
                ls_data, user_id, permissions=permissions)
            kwargs[attr_name] = current

    new_idea = Idea(**kwargs)

    session.add(new_idea)

    if idea_data['parentId']:
        parent = Idea.get_instance(idea_data['parentId'])
    else:
        parent = discussion.root_idea
    session.add(IdeaLink(source=parent, target=new_idea, order=idea_data.get('order', 0.0)))

    session.flush()

    return {'ok': True, '@id': new_idea.uri()}

def test_get_ideas(discussion, test_app, synthesis_1,
                   subidea_1_1_1, test_session):
    all_ideas = test_app.get('/data/Idea')
    assert all_ideas.status_code == 200
    all_ideas = all_ideas.json
    disc_ideas = test_app.get('/data/Discussion/%d/ideas?view=id_only' %
                              (discussion.id,))
    assert disc_ideas.status_code == 200
    disc_ideas = disc_ideas.json
    assert set(all_ideas) == set(disc_ideas)
    synthesis_ideasassocs = test_app.get(
        '/data/Discussion/%d/views/%d/idea_assocs?view=id_only' % (
            discussion.id, synthesis_1.id))
    assert synthesis_ideasassocs.status_code == 200
    synthesis_ideasassocs = synthesis_ideasassocs.json
    syn_ideas = set()
    for assoc_id in synthesis_ideasassocs:
        a = SubGraphIdeaAssociation.get_instance(assoc_id)
        syn_ideas.add(Idea.uri_generic(a.idea_id))
    assert syn_ideas < set(disc_ideas)
    subidea_1_1_1_id = Idea.uri_generic(subidea_1_1_1.id)
    assert subidea_1_1_1_id in disc_ideas
    assert subidea_1_1_1_id not in syn_ideas
    syn_ideas = test_app.get(
        '/data/Discussion/%d/views/%d/ideas?view=id_only' % (
            discussion.id, synthesis_1.id))
    assert syn_ideas.status_code == 200
    syn_ideas = syn_ideas.json
    assert set(syn_ideas) < set(disc_ideas)
    subidea_1_1_1_id = Idea.uri_generic(subidea_1_1_1.id)
    assert subidea_1_1_1_id in disc_ideas
    assert subidea_1_1_1_id not in syn_ideas

def put_extract(request):
    """
    Updating an Extract
    """
    extract_id = request.matchdict['id']
    user_id = authenticated_userid(request)

    updated_extract_data = json.loads(request.body)
    extract = Extract.get_instance(extract_id)
    if not extract:
        raise HTTPNotFound("Extract with id '%s' not found." % extract_id)

    extract.owner_id = user_id or get_database_id("User", extract.owner_id)
    extract.order = updated_extract_data.get('order', extract.order)
    idea_id = updated_extract_data.get('idIdea', None)
    if idea_id:
        idea = Idea.get_instance(idea_id)
        if(idea.get_discussion_id() != extract.get_discussion_id()):
            raise HTTPBadRequest(
                "Extract from discussion %s cannot be associated with an idea from a different discussion." % extract.get_discussion_id())
        extract.idea = idea
    else:
        extract.idea = None

    Extract.db.add(extract)
    #TODO: Merge ranges. Sigh.

    return {'ok': True}

def add_idea_to_synthesis(request):
    """Add an idea to an ExplictSubgraphView"""
    ctx = request.context
    graph_view = ctx.parent_instance
    if isinstance(graph_view, Synthesis) and not graph_view.is_next_synthesis:
        raise HTTPBadRequest("Synthesis is published")
    content = request.json
    idea_id = content.get('@id', None)
    if not idea_id:
        raise HTTPBadRequest("Post an idea with its @id")
    idea = Idea.get_instance(idea_id)
    if not idea:
        raise HTTPNotFound("Unknown idea")
    link = SubGraphIdeaAssociation(idea=idea, sub_graph=graph_view)
    duplicate = link.find_duplicate(False)
    if duplicate:
        link.delete()
        return duplicate.idea.generic_json()
    graph_view.db.add(link)
    graph_view.db.expire(graph_view, ["idea_assocs"])
    graph_view.send_to_changes()
    # special location
    return Response(
        json.dumps(idea.generic_json()), 201, content_type='application/json',
        location=request.url + "/" + str(idea.id))

def post_published_for_bright_mirror_participant(
        request, test_session, discussion, admin_user, participant1_user,
        bright_mirror):
    from assembl.models import Post, Idea, LangString, IdeaRelatedPostLink, PublicationStates
    from graphene.relay import Node
    idea_id = bright_mirror
    raw_id = int(Node.from_global_id(idea_id)[1])
    idea = Idea.get(raw_id)
    p = Post(
        discussion=discussion, creator=participant1_user,
        subject=LangString.create(u"Published by participant"),
        body=LangString.create(u"A simple published fiction by participant"),
        type='post', publication_state=PublicationStates.PUBLISHED,
        message_id="[email protected]",
        creation_date = datetime.utcnow())

    idc = IdeaRelatedPostLink(
        idea=idea,
        creator=admin_user,
        content=p)

    test_session.add(p)
    test_session.add(idc)
    test_session.flush()

    def fin():
        print "finalizer post_published_for_bright_mirror"
        test_session.delete(p)
        test_session.delete(idc)
        test_session.flush()

    request.addfinalizer(fin)
    return p

def voting_widget_view(request):
    user_id = authenticated_userid(request) or Everyone
    ctx = request.context
    view = (request.matchdict or {}).get('view', None)\
        or ctx.get_default_view() or 'default'
    widget = ctx._instance
    permissions = get_permissions(
        user_id, ctx.get_discussion_id())
    json = widget.generic_json(view, user_id, permissions)
    #json['discussion'] = ...
    if user_id != Everyone:
        user = User.get(user_id)
        json['user'] = user.generic_json(view, user_id, permissions)
        json['user_permissions'] = get_permissions(
            user_id, widget.get_discussion_id())
        user_state = widget.get_user_state(user_id)
        if user_state is not None:
            json['user_state'] = user_state
    target_id = request.GET.get('target', None)
    if target_id and Idea.get_database_id(target_id):
        json['user_votes_url'] = widget.get_user_votes_url(target_id)
        json['voting_urls'] = widget.get_voting_urls(target_id)
    json['criteria'] = [idea.generic_json(view, user_id, permissions)
                        for idea in widget.criteria]
    return json

def post_draft_for_bright_mirror(
        request, test_session, discussion, moderator_user,
        bright_mirror):
    from assembl.models import Post, Idea, LangString, IdeaRelatedPostLink, PublicationStates
    from graphene.relay import Node
    idea_id = bright_mirror
    raw_id = int(Node.from_global_id(idea_id)[1])
    idea = Idea.get(raw_id)
    p = Post(
        discussion=discussion, creator=moderator_user,
        subject=LangString.create(u"Draft"),
        body=LangString.create(u"A simple draft fiction"),
        type='post', publication_state=PublicationStates.DRAFT,
        message_id="[email protected]",
        creation_date = datetime.utcnow() - timedelta(days=7))

    idc = IdeaRelatedPostLink(
        idea=idea,
        creator=moderator_user,
        content=p)

    test_session.add(p)
    test_session.add(idc)
    test_session.flush()

    def fin():
        print "finalizer post_draft_for_bright_mirror"
        test_session.delete(p)
        test_session.delete(idc)
        test_session.flush()

    request.addfinalizer(fin)
    return p

def widget_view(request):
    # IF_OWNED not applicable for widgets... so far
    ctx = request.context
    user_id = authenticated_userid(request) or Everyone
    permissions = get_permissions(
        user_id, ctx.get_discussion_id())
    check_permissions(ctx, user_id, permissions, CrudPermissions.READ)
    view = (request.matchdict or {}).get('view', None)\
        or ctx.get_default_view() or 'default'
    json = ctx._instance.generic_json(view, user_id, permissions)
    # json['discussion'] = ...
    if user_id != Everyone:
        user = User.get(user_id)
        user_state = ctx._instance.get_user_state(user_id)
        json['user'] = user.generic_json(view, user_id, permissions)
        json['user_permissions'] = get_permissions(
            user_id, ctx._instance.get_discussion_id())
        if user_state is not None:
            json['user_state'] = user_state
    target_id = request.GET.get('target', None)
    if target_id:
        idea = Idea.get_instance(target_id)
        if idea:
            json['target'] = idea.generic_json(view, user_id, permissions)
        else:
            return HTTPNotFound("No idea "+target_id)
    return json

def get_indexable_contents(session):
    from assembl.models import AgentProfile, Idea, Post
    from assembl.models.post import PublicationStates

    query = session.query(Idea
        ).filter(Idea.tombstone_condition()
        ).filter(Idea.hidden==False
        ).options(
            joinedload(Idea.title).joinedload("entries"),
            joinedload(Idea.synthesis_title).joinedload("entries"),
            joinedload(Idea.description).joinedload("entries")
        )

    for idea in query:
        yield idea

    query = session.query(AgentProfile)
    for user in query:
        yield user

    AllPost = with_polymorphic(Post, '*')
    query = session.query(AllPost
        ).filter(AllPost.tombstone_condition()
        ).filter(AllPost.hidden==False
        ).filter(AllPost.publication_state == PublicationStates.PUBLISHED
        ).options(
            joinedload(AllPost.subject).joinedload("entries"),
            joinedload(AllPost.body).joinedload("entries")
        )
    for post in query:
        for extract in post.extracts:
            yield extract

        yield post

def delete_idea(request):
    idea_id = request.matchdict['id']
    idea = Idea.get_instance(idea_id)

    if not idea:
        raise HTTPNotFound("Idea with id '%s' not found." % idea_id)
    if isinstance(idea, RootIdea):
        raise HTTPBadRequest("Cannot delete root idea.")
    num_childrens = len(idea.children)
    if num_childrens > 0:
        raise HTTPBadRequest("Idea cannot be deleted because it still has %d child ideas." % num_childrens)
    num_extracts = len(idea.extracts)
    if num_extracts > 0:
        raise HTTPBadRequest("Idea cannot be deleted because it still has %d extracts." % num_extracts)
    db = Idea.db()
    db.delete(idea)
    request.response.status = HTTPNoContent.code
    return HTTPNoContent()

def post_ids_of(idea):
    related = text(
        Idea._get_related_posts_statement(),
        bindparams=[bindparam('root_idea_id', idea.id),
                    bindparam('discussion_id', idea.discussion_id)]
        ).columns(column('post_id')).alias('related')
    post_ids = idea.db.query(Content.id).join(
        related, Content.id == related.c.post_id)
    return [x for (x,) in post_ids]

def idea_connexions(request):
    # Returns the ideas this post is connected to.
    # This is a list of tuples of the form:
    # [idea, connexion_idea, connexion_post, extract?]
    # the idea is an ancestor or equal to connexion_idea
    # the connexion_post is an ancestor or equal to current post
    from assembl.models import Idea
    ctx = request.context
    post = ctx._instance
    return Idea.get_idea_ids_showing_post(post.id, True, True)

def create_idea(request):
    discussion_id = int(request.matchdict["discussion_id"])
    session = Discussion.default_db
    discussion = session.query(Discussion).get(int(discussion_id))
    idea_data = json.loads(request.body)

    new_idea = Idea(short_title=idea_data["shortTitle"], long_title=idea_data["longTitle"], discussion=discussion)

    session.add(new_idea)

    if idea_data["parentId"]:
        parent = Idea.get_instance(idea_data["parentId"])
    else:
        parent = discussion.root_idea
    session.add(IdeaLink(source=parent, target=new_idea, order=idea_data.get("order", 0.0)))

    session.flush()

    return {"ok": True, "@id": new_idea.uri()}

def mark_post_read(request):
    discussion_id = int(request.matchdict["discussion_id"])
    discussion = Discussion.get_instance(discussion_id)
    post_id = request.matchdict["id"]
    post = Post.get_instance(post_id)
    if not post:
        raise HTTPNotFound("Post with id '%s' not found." % post_id)
    post_id = post.id
    user_id = authenticated_userid(request)
    if not user_id:
        raise HTTPUnauthorized()
    read_data = json.loads(request.body)
    db = Discussion.db()
    change = False
    with transaction.manager:
        if read_data.get("read", None) is False:
            view = db.query(ViewPost).filter(ViewPost.post_id == post_id).filter(Action.actor_id == user_id).first()
            if view:
                change = True
                db.delete(view)
        else:
            count = db.query(ViewPost).filter(ViewPost.post_id == post_id).filter(Action.actor_id == user_id).count()
            if not count:
                change = True
                db.add(ViewPost(post=post, actor_id=user_id))

    new_counts = []
    if change:
        new_counts = Idea.idea_counts(discussion_id, post_id, user_id)

    return {
        "ok": True,
        "ideas": [
            {
                "@id": Idea.uri_generic(idea_id),
                "@type": db.query(Idea).get(idea_id).external_typename(),
                "num_posts": total_posts,
                "num_read_posts": read_posts,
            }
            for (idea_id, total_posts, read_posts) in new_counts
        ],
    }

def post_to_vote_votables(request):
    ctx = request.context
    target_id = request.POST.get('id', None)
    idea = None
    if target_id:
        idea = Idea.get_instance(target_id)
    if not idea:
        raise HTTPNotFound
    widget = ctx.parent_instance
    widget.add_votable(idea)
    return HTTPOk()  # Not sure this can be called a creation

def get_idea(request):
    idea_id = request.matchdict['id']
    idea = Idea.get_instance(idea_id)
    view_def = request.GET.get('view')

    if not idea:
        raise HTTPNotFound("Idea with id '%s' not found." % idea_id)

    if view_def:
        return idea.generic_json(view_def)
    else:
        return idea.generic_json()

def get_idea(request):
    idea_id = request.matchdict['id']
    idea = Idea.get_instance(idea_id)
    view_def = request.GET.get('view')
    discussion_id = int(request.matchdict['discussion_id'])
    user_id = authenticated_userid(request)
    permissions = get_permissions(user_id, discussion_id)

    if not idea:
        raise HTTPNotFound("Idea with id '%s' not found." % idea_id)

    return idea.generic_json(view_def, user_id, permissions)

def mark_post_read(request):
    """Mark this post as un/read. Return the read post count for all affected ideas."""
    discussion_id = int(request.matchdict['discussion_id'])
    discussion = Discussion.get_instance(discussion_id)
    post_id = request.matchdict['id']
    post = Post.get_instance(post_id)
    if not post:
        raise HTTPNotFound("Post with id '%s' not found." % post_id)
    post_id = post.id
    user_id = request.authenticated_userid
    if not user_id:
        raise HTTPUnauthorized()
    read_data = json.loads(request.body)
    db = discussion.db
    change = False
    with transaction.manager:
        if read_data.get('read', None) is False:
            view = db.query(ViewPost).filter_by(
                post_id=post_id, actor_id=user_id,
                tombstone_date=None).first()
            if view:
                change = True
                view.is_tombstone = True
        else:
            count = db.query(ViewPost).filter_by(
                post_id=post_id, actor_id=user_id,
                tombstone_date=None).count()
            if not count:
                change = True
                db.add(ViewPost(post=post, actor_id=user_id))

    new_counts = []
    if change:
        new_counts = Idea.idea_read_counts(discussion_id, post_id, user_id)

    return { "ok": True, "ideas": [
        {"@id": Idea.uri_generic(idea_id),
         "num_read_posts": read_posts
        } for (idea_id, read_posts) in new_counts] }

def get_idea_extracts(request):
    idea_id = request.matchdict["id"]
    idea = Idea.get_instance(idea_id)
    view_def = request.GET.get("view") or "default"
    discussion_id = int(request.matchdict["discussion_id"])
    user_id = authenticated_userid(request) or Everyone
    permissions = get_permissions(user_id, discussion_id)

    if not idea:
        raise HTTPNotFound("Idea with id '%s' not found." % idea_id)

    extracts = Extract.default_db.query(Extract).filter(Extract.idea_id == idea.id).order_by(Extract.order.desc())

    return [extract.generic_json(view_def, user_id, permissions) for extract in extracts]